├── .idea ├── ant.xml ├── copyright │ └── profiles_settings.xml ├── libraries │ ├── build.xml │ ├── build2.xml │ ├── dist.xml │ └── dist2.xml └── vcs.xml ├── Disk-backed-map.iml ├── README ├── build.xml ├── docs └── README ├── examples └── com │ └── alok │ └── diskmap │ └── cli │ └── Main.java ├── lib ├── build │ ├── junit-addons.jar │ └── junit.jar ├── dist │ ├── hessian-io.jar │ ├── jetty-6.1.22.jar │ ├── jetty-util-6.1.22.jar │ └── servlet-api-2.5-20081211.jar └── src │ └── hessian-io-src.jar ├── src └── com │ └── alok │ └── diskmap │ ├── Configuration.java │ ├── ConversionUtils.java │ ├── DiskBackedMap.java │ ├── Node.java │ ├── Page.java │ ├── RBTree.java │ ├── Record.java │ ├── ZipMap.java │ ├── io │ ├── BaseDiskIO.java │ ├── BlockingDiskIO.java │ ├── DiskIO.java │ └── NonBlockingDiskIO.java │ └── utils │ ├── DefaultObjectConverter.java │ ├── Hessian2ObjectConverter.java │ └── ObjectConverter.java └── test └── com └── alok └── diskmap ├── BasicOpsTest.java ├── ConcurrentOpsTest.java ├── ConversionUtilsTest.java ├── PageTest.java ├── RBTreeTest.java ├── RecordTest.java ├── ZipMapTest.java ├── mem └── MemoryUsageTest.java ├── mock ├── MockObjectWithBinaryData.java └── MockSimpleObject.java ├── perf ├── Config.java ├── Driver.java ├── KeyValueGen.java ├── PerfTask.java ├── Reader.java └── StatsCollector.java └── utils └── ObjectConversionTest.java /.idea/ant.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/copyright/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /.idea/libraries/build.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /.idea/libraries/build2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /.idea/libraries/dist.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /.idea/libraries/dist2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /Disk-backed-map.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | A small library that provide a disk backed map implementation for storing large number of key value pairs. The map implementations (HashMap, HashTable) max out around 3-4Million keys/GB of memory for very simple key/value pairs and in most cases the limit is much lower. DiskBacked map on the other hand can store betweeen 16Million (64bit JVM) to 20Million(32bit JVM) keys/GB, regardless the size of the key/value pairs. 2 | -------------------------------------------------------------------------------- /build.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /docs/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aloksingh/disk-backed-map/5d048d846c82f9d1559790c05b6f7dda3a15fb50/docs/README -------------------------------------------------------------------------------- /examples/com/alok/diskmap/cli/Main.java: -------------------------------------------------------------------------------- 1 | package com.alok.diskmap.cli; 2 | 3 | import com.alok.diskmap.DiskBackedMap; 4 | import org.mortbay.jetty.Server; 5 | import org.mortbay.jetty.servlet.Context; 6 | import org.mortbay.jetty.servlet.ServletHolder; 7 | 8 | import javax.servlet.*; 9 | import javax.servlet.http.HttpServlet; 10 | import javax.servlet.http.HttpServletRequest; 11 | import javax.servlet.http.HttpServletResponse; 12 | import java.io.File; 13 | import java.io.IOException; 14 | import java.util.Map; 15 | 16 | public class Main { 17 | public static void main(String[] args) throws Exception { 18 | Server server = new Server(8080); 19 | Context root = new Context(server,"/", Context.SESSIONS); 20 | root.addServlet(new ServletHolder(new HelloServlet()), "/*"); 21 | server.start(); 22 | 23 | } 24 | 25 | public static class HelloServlet extends HttpServlet { 26 | 27 | @Override 28 | public void init(ServletConfig servletConfig) throws ServletException { 29 | File tempFile = null; 30 | try { 31 | tempFile = File.createTempFile("foo", "tmp"); 32 | servletConfig.getServletContext().setAttribute("storage", new DiskBackedMap(System.getProperty("java.io.tmpdir"))); 33 | } catch (IOException e) { 34 | e.printStackTrace(); 35 | throw new ServletException("Cannot create temp file", e); 36 | } 37 | } 38 | 39 | @Override 40 | protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { 41 | String uri = req.getRequestURI(); 42 | String key = ""; 43 | String value = ""; 44 | if(uri.indexOf("/map/") > -1){ 45 | key = uri.substring(uri.indexOf("/map/") + 5); 46 | Map map = (Map) req.getSession().getServletContext().getAttribute("storage"); 47 | value = (String) map.get(key); 48 | 49 | } 50 | resp.getWriter().format(""); 54 | } 55 | 56 | @Override 57 | protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { 58 | String key = req.getParameter("key"); 59 | String value = req.getParameter("value"); 60 | Map map = (Map) req.getSession().getServletContext().getAttribute("storage"); 61 | map.put(key, value); 62 | resp.getWriter().format("ok"); 63 | } 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /lib/build/junit-addons.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aloksingh/disk-backed-map/5d048d846c82f9d1559790c05b6f7dda3a15fb50/lib/build/junit-addons.jar -------------------------------------------------------------------------------- /lib/build/junit.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aloksingh/disk-backed-map/5d048d846c82f9d1559790c05b6f7dda3a15fb50/lib/build/junit.jar -------------------------------------------------------------------------------- /lib/dist/hessian-io.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aloksingh/disk-backed-map/5d048d846c82f9d1559790c05b6f7dda3a15fb50/lib/dist/hessian-io.jar -------------------------------------------------------------------------------- /lib/dist/jetty-6.1.22.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aloksingh/disk-backed-map/5d048d846c82f9d1559790c05b6f7dda3a15fb50/lib/dist/jetty-6.1.22.jar -------------------------------------------------------------------------------- /lib/dist/jetty-util-6.1.22.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aloksingh/disk-backed-map/5d048d846c82f9d1559790c05b6f7dda3a15fb50/lib/dist/jetty-util-6.1.22.jar -------------------------------------------------------------------------------- /lib/dist/servlet-api-2.5-20081211.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aloksingh/disk-backed-map/5d048d846c82f9d1559790c05b6f7dda3a15fb50/lib/dist/servlet-api-2.5-20081211.jar -------------------------------------------------------------------------------- /lib/src/hessian-io-src.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aloksingh/disk-backed-map/5d048d846c82f9d1559790c05b6f7dda3a15fb50/lib/src/hessian-io-src.jar -------------------------------------------------------------------------------- /src/com/alok/diskmap/Configuration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap; 18 | 19 | import java.io.File; 20 | 21 | public class Configuration { 22 | private int flushInterval; 23 | private File dir; 24 | private int number; 25 | private int readerPoolSize = 3; 26 | private boolean useNonBlockingReader = true; 27 | 28 | public Configuration() { 29 | } 30 | 31 | public Configuration(Configuration cfg) { 32 | this.flushInterval = cfg.getFlushInterval(); 33 | this.dir = new File(cfg.getDataDir().getAbsolutePath()); 34 | this.number = cfg.getNumber(); 35 | this.readerPoolSize = cfg.getReaderPoolSize(); 36 | this.useNonBlockingReader = cfg.getUseNonBlockingReader(); 37 | } 38 | 39 | public Configuration setFlushInterval(int interval) { 40 | this.flushInterval = interval; 41 | return this; 42 | } 43 | 44 | public int getFlushInterval() { 45 | return flushInterval; 46 | } 47 | 48 | public Configuration setDataDir(File dir) { 49 | this.dir = dir; 50 | return this; 51 | } 52 | 53 | public File getDataDir() { 54 | return dir; 55 | } 56 | 57 | public Configuration setNumber(int number) { 58 | this.number = number; 59 | return this; 60 | } 61 | 62 | public int getNumber() { 63 | return number; 64 | } 65 | 66 | public String getDataFileName(String extension){ 67 | return getDataDir().getAbsolutePath() + File.separator + getNumber() + "." + extension; 68 | } 69 | 70 | public int getReaderPoolSize() { 71 | return readerPoolSize; 72 | } 73 | 74 | public boolean getUseNonBlockingReader() { 75 | return useNonBlockingReader; 76 | } 77 | 78 | public Configuration setUseNonBlockingReader(boolean useNonBlockingReader) { 79 | this.useNonBlockingReader = useNonBlockingReader; 80 | return this; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/com/alok/diskmap/ConversionUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap; 18 | 19 | import com.alok.diskmap.utils.DefaultObjectConverter; 20 | import com.alok.diskmap.utils.ObjectConverter; 21 | 22 | import java.io.Serializable; 23 | 24 | public class ConversionUtils { 25 | private static final int poly = 0x1021; 26 | 27 | private static final int[] crcTable = new int[256]; 28 | 29 | static { 30 | // initialise scrambler table 31 | for (int i = 0; i < 256; i++) { 32 | int fcs = 0; 33 | int d = i << 8; 34 | for (int k = 0; k < 8; k++) { 35 | if (((fcs ^ d) & 0x8000) != 0) { 36 | fcs = (fcs << 1) ^ poly; 37 | } else { 38 | fcs = (fcs << 1); 39 | } 40 | d <<= 1; 41 | fcs &= 0xffff; 42 | } 43 | crcTable[i] = fcs; 44 | } 45 | } 46 | 47 | public static final ConversionUtils instance = new ConversionUtils(); 48 | 49 | private ObjectConverter os; 50 | 51 | public ConversionUtils() { 52 | try{ 53 | os = new DefaultObjectConverter(); 54 | }catch(Exception e){ 55 | System.err.println("Unable to create hessian object convertor, using the default convertor."); 56 | os = new DefaultObjectConverter(); 57 | } 58 | } 59 | 60 | public byte[] intToBytes(int n) { 61 | byte[] b = new byte[4]; 62 | for (int i = 0; i < b.length; i++) { 63 | b[3 - i] = (byte) (n >>> (i * 8)); 64 | } 65 | return b; 66 | } 67 | 68 | public byte[] longToBytes(long n) { 69 | byte[] b = new byte[8]; 70 | for (int i = 0; i < b.length; i++) { 71 | b[b.length - 1 - i] = (byte) (n >>> (i * 8)); 72 | } 73 | return b; 74 | } 75 | 76 | public int byteToInt(byte[] b) { 77 | return byteToInt(b, 0); 78 | } 79 | public int byteToInt(byte[] b, int offset) { 80 | int n = 0; 81 | for (int i = offset; i < offset + 4; i++) { 82 | n <<= 8; 83 | n ^= (int) b[i] & 0xFF; 84 | } 85 | return n; 86 | } 87 | 88 | public long byteToLong(byte[] b) { 89 | long n = 0; 90 | for (int i = 0; i < 8; i++) { 91 | n <<= 8; 92 | n ^= (long) b[i] & 0xFF; 93 | } 94 | return n; 95 | } 96 | 97 | public byte[] serialize(Serializable object) throws Exception { 98 | return os.serialize(object); 99 | } 100 | 101 | public T deserialize(byte[] buffer) { 102 | return (T) os.deserialize(buffer); 103 | } 104 | 105 | public byte[] shortToBytes(int n) { 106 | return shortToBytes((short) n); 107 | } 108 | public byte[] shortToBytes(short n) { 109 | byte[] b = new byte[2]; 110 | for (int i = 0; i < b.length; i++) { 111 | b[1 - i] = (byte) (n >>> (i * 8)); 112 | } 113 | return b; 114 | } 115 | 116 | public short byteToShort(byte[] b) { 117 | return byteToShort(b, 0); 118 | } 119 | public short byteToShort(byte[] b, int offset) { 120 | short n = 0; 121 | for (int i = offset; i < offset + 2; i++) { 122 | n <<= 8; 123 | n ^= (int) b[i] & 0xFF; 124 | } 125 | return n; 126 | } 127 | 128 | public static short crc16(byte[] ba) { 129 | int work = 0xffff; 130 | for (byte b : ba) { 131 | work = (crcTable[(b ^ (work >>> 8)) & 0xff] ^ (work << 8)) & 132 | 0xffff; 133 | } 134 | return (short) work; 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /src/com/alok/diskmap/DiskBackedMap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap; 18 | 19 | import java.io.Closeable; 20 | import java.io.File; 21 | import java.io.IOException; 22 | import java.io.Serializable; 23 | import java.util.*; 24 | import java.util.logging.Level; 25 | import java.util.logging.Logger; 26 | 27 | public class DiskBackedMap implements Map, Closeable { 28 | private Logger log = Logger.getLogger(DiskBackedMap.class.getName()); 29 | private Store store; 30 | 31 | public DiskBackedMap(String dataDir) { 32 | this.store = new Store(new Configuration().setDataDir(new File(dataDir))); 33 | } 34 | 35 | public DiskBackedMap(Configuration config) { 36 | this.store = new Store(config); 37 | } 38 | 39 | @Override 40 | public void clear() { 41 | store.clear(); 42 | } 43 | 44 | @Override 45 | public boolean containsKey(Object key) { 46 | return store.get((K) key) != null; 47 | } 48 | 49 | @Override 50 | public boolean containsValue(Object value) { 51 | throw new UnsupportedOperationException(); 52 | } 53 | 54 | @Override 55 | public Set> entrySet() { 56 | return null; 57 | } 58 | 59 | @Override 60 | @SuppressWarnings("element-type-mismatch") 61 | public V get(Object key) { 62 | return store.get((K) key); 63 | } 64 | 65 | @Override 66 | public boolean isEmpty() { 67 | return store.size() == 0; 68 | } 69 | 70 | @Override 71 | public Set keySet() { 72 | throw new UnsupportedOperationException(); 73 | } 74 | 75 | @Override 76 | public V put(K key, V value) { 77 | return store.save(key, value); 78 | } 79 | 80 | @Override 81 | public void putAll(Map m) { 82 | for (K key : m.keySet()) { 83 | put(key, m.get(key)); 84 | } 85 | } 86 | 87 | @Override 88 | public V remove(Object key) { 89 | V value = store.get((K) key); 90 | store.remove((K) key); 91 | return value; 92 | } 93 | 94 | @Override 95 | public int size() { 96 | return store.size(); 97 | } 98 | 99 | @Override 100 | public Collection values() { 101 | throw new UnsupportedOperationException(); 102 | } 103 | 104 | public long sizeOnDisk(){ 105 | return store.sizeOnDisk(); 106 | } 107 | 108 | public void close() throws IOException { 109 | store.close(); 110 | } 111 | 112 | public void gc() throws Exception { 113 | store.vacuum(); 114 | } 115 | 116 | @Override 117 | public void finalize() throws Throwable { 118 | this.close(); 119 | super.finalize(); 120 | } 121 | 122 | public class Store implements Closeable { 123 | private List> pages; 124 | private int magicNumber = 13; 125 | 126 | public Store(Configuration cfg) { 127 | init(cfg); 128 | } 129 | 130 | private void init(Configuration cfg) { 131 | pages = new ArrayList>(magicNumber); 132 | for (int i = 0; i < magicNumber; i++) { 133 | Configuration config = new Configuration(cfg); 134 | config.setNumber(i); 135 | pages.add(new Page(config)); 136 | } 137 | } 138 | 139 | public V save(K key, V value) { 140 | Page kvPage = findPage(key); 141 | return kvPage.save(key, value); 142 | } 143 | 144 | public V get(K key) { 145 | return findPage(key).load(key); 146 | } 147 | 148 | private Page findPage(K key) { 149 | int idx = key.hashCode() % magicNumber; 150 | return pages.get(Math.abs(idx)); 151 | } 152 | 153 | private void remove(K key) { 154 | findPage(key).remove(key); 155 | } 156 | 157 | private int size() { 158 | int size = 0; 159 | for (Page page : pages) { 160 | size = page.keyCount(); 161 | } 162 | return size; 163 | } 164 | 165 | public synchronized void close() { 166 | for (Page page : pages) { 167 | page.close(); 168 | } 169 | } 170 | 171 | public void vacuum() throws Exception { 172 | log.log(Level.INFO, "Starting gc process"); 173 | long time = 0; 174 | for (Page page : pages) { 175 | long pTime = System.currentTimeMillis(); 176 | log.log(Level.INFO, "Started Vacuuming page:" + page.toString()); 177 | page.vacuum(); 178 | pTime = System.currentTimeMillis() - pTime; 179 | log.log(Level.INFO, "Completed Vacuuming page in :" + pTime + " ms"); 180 | time += pTime; 181 | } 182 | log.log(Level.INFO, "Vacuum Complete:" + time + " ms"); 183 | } 184 | 185 | public long sizeOnDisk() { 186 | long size = 0; 187 | for (Page page : pages) { 188 | size = page.size(); 189 | } 190 | return size; 191 | } 192 | 193 | public synchronized void clear() { 194 | for (Page page : pages) { 195 | page.clear(); 196 | } 197 | } 198 | } 199 | } -------------------------------------------------------------------------------- /src/com/alok/diskmap/Node.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap; 18 | 19 | import java.io.Externalizable; 20 | import java.io.IOException; 21 | import java.io.ObjectInput; 22 | import java.io.ObjectOutput; 23 | 24 | public class Node implements Externalizable 25 | { 26 | public int key; 27 | private long value; 28 | private long[] values; 29 | public Node left; 30 | public Node right; 31 | public Node parent; 32 | public int color; 33 | 34 | public Node() { 35 | } 36 | 37 | public void writeExternal(ObjectOutput objectOutput) throws IOException { 38 | objectOutput.writeInt(key); 39 | objectOutput.writeLong(value); 40 | if(values != null){ 41 | objectOutput.writeInt(values.length); 42 | for (long l : values) { 43 | objectOutput.writeLong(l); 44 | } 45 | }else{ 46 | objectOutput.writeInt(-1); 47 | } 48 | if(right != null){ 49 | objectOutput.writeInt(1); 50 | right.writeExternal(objectOutput); 51 | }else { 52 | objectOutput.writeInt(-1); 53 | } 54 | if(left != null){ 55 | objectOutput.writeInt(1); 56 | left.writeExternal(objectOutput); 57 | }else { 58 | objectOutput.writeInt(-1); 59 | } 60 | } 61 | 62 | public void readExternal(ObjectInput objectInput) throws IOException, ClassNotFoundException { 63 | key = objectInput.readInt(); 64 | value = objectInput.readLong(); 65 | int valLength = objectInput.readInt(); 66 | if(valLength > 0){ 67 | values = new long[valLength]; 68 | for (int i = 0; i implements Closeable { 32 | private static final Logger logger = Logger.getLogger(Page.class.getName()); 33 | private static final boolean DEBUG = false; 34 | private RBTree layout; 35 | private final Configuration cfg; 36 | private DiskIO io; 37 | private ConversionUtils cUtils = ConversionUtils.instance; 38 | 39 | private ReadWriteLock rwl = new ReentrantReadWriteLock(); 40 | 41 | 42 | public Page(File dir, int number) { 43 | this(new Configuration().setFlushInterval(1000).setDataDir(dir).setNumber(number)); 44 | } 45 | 46 | public Page(Configuration cfg) { 47 | this.cfg = cfg; 48 | layout = new RBTree(); 49 | this.io = this.cfg.getUseNonBlockingReader() ? new NonBlockingDiskIO(cfg) : new BlockingDiskIO(cfg); 50 | loadData(io); 51 | } 52 | 53 | public V load(K key) { 54 | this.rwl.readLock().lock(); 55 | try{ 56 | Record record = loadRecord(key); 57 | return record == null ? null : cUtils.deserialize(record.getValue()); 58 | }catch(Exception e){ 59 | logger.log(Level.SEVERE, String.format("%s load([%s]) failed", cfg.getDataFileName("dat"), String.valueOf(key)), e); 60 | throw new RuntimeException(e); 61 | }finally { 62 | this.rwl.readLock().unlock(); 63 | } 64 | } 65 | 66 | private Record loadRecord(Serializable key) { 67 | long[] locations = layout.lookup(key.hashCode()); 68 | try { 69 | if (locations != null) { 70 | if (locations.length == 1) { 71 | Record record = io.lookup(locations[0]); 72 | return cUtils.deserialize(record.getKey()).equals(key) ? record : null; 73 | } else { 74 | for (long location : locations) { 75 | Record r = io.lookup(location); 76 | if (key.equals(cUtils.deserialize(r.getKey()))) { 77 | return r; 78 | } 79 | } 80 | } 81 | } 82 | } catch (Exception e) { 83 | logger.log(Level.SEVERE, String.format("%s loadRecord([%s]) failed. Locations [%s]", this.cfg.getDataFileName(".dat"), String.valueOf(key), Arrays.toString(locations)), e); 84 | throw new RuntimeException(e); 85 | } 86 | return null; 87 | } 88 | 89 | public V save(K key, V value) { 90 | rwl.writeLock().lock(); 91 | log(Level.INFO, "[%s] save([%s], [%s]) started", cfg.getDataFileName("dat"), key, value); 92 | try { 93 | byte[] kBuffer = cUtils.serialize(key); 94 | byte[] vBuffer = cUtils.serialize(value); 95 | Record r = new Record(kBuffer, vBuffer, Record.ACTIVE, key.hashCode(), -1); 96 | long location = io.write(r); 97 | //Check to see if a old record exists 98 | Record oldRecord = loadRecord(key); 99 | if(oldRecord != null){ 100 | layout.delete(oldRecord.getHash(), oldRecord.getLocation()); 101 | } 102 | updateLayout(r, location); 103 | r.setLocation(location); 104 | if(oldRecord != null){ 105 | oldRecord.setFlag(Record.DELETED); 106 | io.update(oldRecord, r); 107 | }else{ 108 | io.update(r); 109 | } 110 | log(Level.INFO, "[%s] save([%s], [%s]) complete. Record[%s]", cfg.getDataFileName("dat"), key, value, r); 111 | return (V) cUtils.deserialize(vBuffer); 112 | } catch (Exception e) { 113 | logger.log(Level.SEVERE, String.format("[%s] save([%s], [%s]) failed", cfg.getDataFileName("dat"), String.valueOf(key), String.valueOf(value)), e); 114 | throw new RuntimeException(e); 115 | }finally { 116 | rwl.writeLock().unlock(); 117 | } 118 | } 119 | 120 | private void log(Level level, String msg, Object...args) { 121 | if(logger.isLoggable(level)){ 122 | String[] sArgs = new String[args.length]; 123 | for (int i = 0; i < args.length; i++) { 124 | sArgs[i] = String.valueOf(args[i]); 125 | } 126 | if(level.equals(Level.INFO)){ 127 | if(DEBUG){ 128 | logger.log(level, String.format(msg, args)); 129 | } 130 | }else{ 131 | logger.log(level, msg); 132 | } 133 | } 134 | } 135 | 136 | public void remove(K key) { 137 | rwl.writeLock().lock(); 138 | try{ 139 | Record oldRecord = loadRecord(key); 140 | if(oldRecord != null){ 141 | layout.delete(oldRecord.getHash(), oldRecord.getLocation()); 142 | oldRecord.setFlag(Record.DELETED); 143 | io.update(oldRecord); 144 | } 145 | }catch(Exception e){ 146 | logger.log(Level.SEVERE, String.format("[%s] remove([%s]) failed", cfg.getDataFileName("dat"), String.valueOf(key)), e); 147 | }finally { 148 | rwl.writeLock().unlock(); 149 | } 150 | } 151 | 152 | 153 | private List lookup(long[] locations) throws IOException { 154 | List records = new ArrayList(locations.length); 155 | for (Long location : locations) { 156 | Record r = io.lookup(location); 157 | records.add(r); 158 | } 159 | return records; 160 | } 161 | 162 | private void updateLayout(Record r, long location) { 163 | layout.insert(r.getHash(), location); 164 | } 165 | 166 | public Iterator> iterator() { 167 | return new Iterator>() { 168 | private Node current = layout.root; 169 | private List records = new ArrayList(); 170 | 171 | public boolean hasNext() { 172 | loadNext(); 173 | return records.size() > 0; 174 | } 175 | 176 | public Entry next() { 177 | try { 178 | final K key = cUtils.deserialize(records.get(0).getKey()); 179 | final V value = cUtils.deserialize(records.get(0).getValue()); 180 | records.remove(0); 181 | return new Entry() { 182 | public K getKey() { 183 | return key; 184 | } 185 | 186 | public V getValue() { 187 | return value; 188 | } 189 | 190 | public V setValue(V value) { 191 | throw new UnsupportedOperationException("Not supported yet."); 192 | } 193 | }; 194 | } catch (Exception e) { 195 | throw new RuntimeException(e); 196 | } 197 | } 198 | 199 | public void remove() { 200 | throw new UnsupportedOperationException("Not supported yet."); 201 | } 202 | 203 | private void loadNext() { 204 | try { 205 | if (records.size() == 0 && current != null) { 206 | if (current.getValues() == null) { 207 | records.add(io.lookup(current.getValue())); 208 | } else { 209 | records.addAll(lookup(current.getValues())); 210 | } 211 | if (current.left != null) { 212 | current = current.left; 213 | } else if (current.right != null) { 214 | current = current.right; 215 | } else { 216 | current = null; 217 | } 218 | } 219 | } catch (Exception e) { 220 | throw new RuntimeException(e); 221 | } 222 | } 223 | 224 | }; 225 | } 226 | 227 | private void loadData(DiskIO io) { 228 | rwl.writeLock().lock(); 229 | try{ 230 | log(Level.INFO, "%s loadData started", cfg.getDataFileName("dat")); 231 | long time = System.currentTimeMillis(); 232 | int count = 0; 233 | for (Record r : io) { 234 | if (r.getFlag() == Record.ACTIVE && r.getLocation() != -1) { 235 | layout.insert(r.getHash(), r.getLocation()); 236 | } 237 | count++; 238 | } 239 | log(Level.INFO, "%s loadData loadData complete. Items: %s in ms: %s", count, (System.currentTimeMillis() - time),cfg.getDataFileName("dat")); 240 | }catch(Exception e){ 241 | log(Level.SEVERE, String.format("%s loadData failed", cfg.getDataFileName("dat"))); 242 | }finally { 243 | rwl.writeLock().unlock(); 244 | } 245 | } 246 | 247 | public void vacuum() throws Exception { 248 | rwl.writeLock().lock(); 249 | try{ 250 | log(Level.INFO, "%s vaccum/gc started", cfg.getDataFileName("dat")); 251 | io.vacuum(new CurrentRecordFilter(layout)); 252 | log(Level.INFO, "%s vaccum/gc complete", cfg.getDataFileName("dat")); 253 | }catch(Exception e){ 254 | log(Level.SEVERE, String.format("%s vaccum/gc failed", cfg.getDataFileName("dat"))); 255 | }finally { 256 | rwl.writeLock().unlock(); 257 | } 258 | } 259 | 260 | 261 | public void close() { 262 | io.close(); 263 | } 264 | 265 | public long size() { 266 | return this.io.size(); 267 | } 268 | 269 | public int keyCount() { 270 | return this.layout.count(); 271 | } 272 | 273 | public void clear() { 274 | rwl.writeLock().lock(); 275 | try{ 276 | log(Level.INFO, "%s clearing", cfg.getDataFileName("dat")); 277 | io.clear(); 278 | this.layout = new RBTree(); 279 | log(Level.INFO, "%s cleared", cfg.getDataFileName("dat")); 280 | }catch(Exception e){ 281 | log(Level.SEVERE, String.format("%s clearing failed", cfg.getDataFileName("dat"))); 282 | }finally { 283 | rwl.writeLock().unlock(); 284 | } 285 | } 286 | 287 | private class CurrentRecordFilter implements DiskIO.RecordFilter{ 288 | private RBTree layout; 289 | 290 | public CurrentRecordFilter(RBTree layout){ 291 | this.layout = layout; 292 | } 293 | 294 | @Override 295 | public boolean accept(Record r) { 296 | long[] locations = layout.lookup(r.getHash()); 297 | if (locations == null || locations.length == 0) { 298 | return false; 299 | } 300 | for (long location : locations) { 301 | if (location == r.getLocation()) { 302 | return true; 303 | } 304 | } 305 | return false; 306 | } 307 | 308 | @Override 309 | public void update(Record r, long newLocation) { 310 | rwl.writeLock().lock(); 311 | try{ 312 | log(Level.INFO, "%s update started", cfg.getDataFileName("dat")); 313 | layout.delete(r.getHash(), r.getLocation()); 314 | layout.insert(r.getHash(), newLocation); 315 | }catch(Exception e){ 316 | log(Level.SEVERE, String.format("%s update failed", cfg.getDataFileName("dat"))); 317 | }finally { 318 | rwl.writeLock().unlock(); 319 | } 320 | } 321 | } 322 | 323 | @Override 324 | public String toString() { 325 | return "Page{" + 326 | "data=" + cfg.getDataFileName("dat")+ 327 | '}'; 328 | } 329 | } 330 | -------------------------------------------------------------------------------- /src/com/alok/diskmap/RBTree.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap; 18 | 19 | import com.alok.diskmap.Node.Color; 20 | 21 | import java.io.Externalizable; 22 | import java.io.IOException; 23 | import java.io.ObjectInput; 24 | import java.io.ObjectOutput; 25 | 26 | public class RBTree implements Externalizable { 27 | private static final int INDENT_STEP = 4; 28 | 29 | public Node root; 30 | 31 | public RBTree() { 32 | root = null; 33 | } 34 | 35 | private static int nodeColor(Node n) { 36 | return n == null ? Color.BLACK : n.color; 37 | } 38 | 39 | private Node lookupNode(int key) { 40 | Node n = root; 41 | while (n != null) { 42 | int compResult = compare(key, n.key); 43 | if (compResult == 0) { 44 | return n; 45 | } else if (compResult < 0) { 46 | n = n.left; 47 | } else { 48 | assert compResult > 0; 49 | n = n.right; 50 | } 51 | } 52 | return n; 53 | } 54 | 55 | private static int compare(int n1, int n2) { 56 | return (n1 < n2 ? -1 : (n1 == n2 ? 0 : 1)); 57 | } 58 | 59 | public long[] lookup(int key) { 60 | return _lookup(rehash(key)); 61 | } 62 | 63 | private long[] _lookup(int key) { 64 | Node n = lookupNode(key); 65 | if (n == null) { 66 | return null; 67 | } 68 | if (n.getValues() != null) { 69 | return n.getValues(); 70 | } 71 | return new long[]{n.getValue()}; 72 | } 73 | 74 | private void rotateLeft(Node n) { 75 | Node r = n.right; 76 | replaceNode(n, r); 77 | n.right = r.left; 78 | if (r.left != null) { 79 | r.left.parent = n; 80 | } 81 | r.left = n; 82 | n.parent = r; 83 | } 84 | 85 | private void rotateRight(Node n) { 86 | Node l = n.left; 87 | replaceNode(n, l); 88 | n.left = l.right; 89 | if (l.right != null) { 90 | l.right.parent = n; 91 | } 92 | l.right = n; 93 | n.parent = l; 94 | } 95 | 96 | private void replaceNode(Node oldn, Node newn) { 97 | if (oldn.parent == null) { 98 | root = newn; 99 | } else { 100 | if (oldn == oldn.parent.left) 101 | oldn.parent.left = newn; 102 | else 103 | oldn.parent.right = newn; 104 | } 105 | if (newn != null) { 106 | newn.parent = oldn.parent; 107 | } 108 | } 109 | 110 | public void insert(int key, long value) { 111 | _insert(rehash(key), value); 112 | } 113 | 114 | private void _insert(int key, long value) { 115 | Node insertedNode = new Node(key, value, Color.RED, null, null); 116 | if (root == null) { 117 | root = insertedNode; 118 | } else { 119 | Node n = root; 120 | while (true) { 121 | int compResult = compare(key, n.key); 122 | if (compResult == 0) { 123 | if (n.getValue() == value) { 124 | return; 125 | } else if (n.getValues() != null) { 126 | n.addValue(value); 127 | } else { 128 | //hash collision 129 | n.addValue(n.getValue()); 130 | n.addValue(value); 131 | n.setValue(-1); 132 | } 133 | return; 134 | } else if (compResult < 0) { 135 | if (n.left == null) { 136 | n.left = insertedNode; 137 | break; 138 | } else { 139 | n = n.left; 140 | } 141 | } else { 142 | assert compResult > 0; 143 | if (n.right == null) { 144 | n.right = insertedNode; 145 | break; 146 | } else { 147 | n = n.right; 148 | } 149 | } 150 | } 151 | insertedNode.parent = n; 152 | } 153 | insertCase1(insertedNode); 154 | } 155 | 156 | private void insertCase1(Node n) { 157 | if (n.parent == null) 158 | n.color = Color.BLACK; 159 | else 160 | insertCase2(n); 161 | } 162 | 163 | private void insertCase2(Node n) { 164 | if (nodeColor(n.parent) == Color.BLACK) 165 | return; // Tree is still valid 166 | else 167 | insertCase3(n); 168 | } 169 | 170 | void insertCase3(Node n) { 171 | if (nodeColor(n.uncle()) == Color.RED) { 172 | n.parent.color = Color.BLACK; 173 | n.uncle().color = Color.BLACK; 174 | n.grandparent().color = Color.RED; 175 | insertCase1(n.grandparent()); 176 | } else { 177 | insertCase4(n); 178 | } 179 | } 180 | 181 | void insertCase4(Node n) { 182 | if (n == n.parent.right && n.parent == n.grandparent().left) { 183 | rotateLeft(n.parent); 184 | n = n.left; 185 | } else if (n == n.parent.left && n.parent == n.grandparent().right) { 186 | rotateRight(n.parent); 187 | n = n.right; 188 | } 189 | insertCase5(n); 190 | } 191 | 192 | void insertCase5(Node n) { 193 | n.parent.color = Color.BLACK; 194 | n.grandparent().color = Color.RED; 195 | if (n == n.parent.left && n.parent == n.grandparent().left) { 196 | rotateRight(n.grandparent()); 197 | } else { 198 | assert n == n.parent.right && n.parent == n.grandparent().right; 199 | rotateLeft(n.grandparent()); 200 | } 201 | } 202 | 203 | public void delete(int key, long value) { 204 | _delete(rehash(key), value); 205 | } 206 | 207 | private void _delete(int key, long value) { 208 | Node n = lookupNode(key); 209 | if (n == null) 210 | return; // Key not found, do nothing 211 | if (n.getValues() != null) { 212 | n.deleteValue(value); 213 | return; 214 | } 215 | if (n.left != null && n.right != null) { 216 | // Copy key/value from predecessor and then delete it instead 217 | Node pred = maximumNode(n.left); 218 | n.key = pred.key; 219 | if (pred.getValues() == null) { 220 | n.setValue(pred.getValue()); 221 | } else { 222 | n.setValues(pred.getValues()); 223 | } 224 | n = pred; 225 | } 226 | 227 | assert n.left == null || n.right == null; 228 | Node child = (n.right == null) ? n.left : n.right; 229 | if (nodeColor(n) == Color.BLACK) { 230 | n.color = nodeColor(child); 231 | deleteCase1(n); 232 | } 233 | replaceNode(n, child); 234 | } 235 | 236 | private static Node maximumNode(Node n) { 237 | assert n != null; 238 | while (n.right != null) { 239 | n = n.right; 240 | } 241 | return n; 242 | } 243 | 244 | private void deleteCase1(Node n) { 245 | if (n.parent == null) 246 | return; 247 | else 248 | deleteCase2(n); 249 | } 250 | 251 | private void deleteCase2(Node n) { 252 | if (nodeColor(n.sibling()) == Color.RED) { 253 | n.parent.color = Color.RED; 254 | n.sibling().color = Color.BLACK; 255 | if (n == n.parent.left) 256 | rotateLeft(n.parent); 257 | else 258 | rotateRight(n.parent); 259 | } 260 | deleteCase3(n); 261 | } 262 | 263 | private void deleteCase3(Node n) { 264 | if (nodeColor(n.parent) == Color.BLACK && 265 | nodeColor(n.sibling()) == Color.BLACK && 266 | nodeColor(n.sibling().left) == Color.BLACK && 267 | nodeColor(n.sibling().right) == Color.BLACK) { 268 | n.sibling().color = Color.RED; 269 | deleteCase1(n.parent); 270 | } else 271 | deleteCase4(n); 272 | } 273 | 274 | private void deleteCase4(Node n) { 275 | if (nodeColor(n.parent) == Color.RED && 276 | nodeColor(n.sibling()) == Color.BLACK && 277 | nodeColor(n.sibling().left) == Color.BLACK && 278 | nodeColor(n.sibling().right) == Color.BLACK) { 279 | n.sibling().color = Color.RED; 280 | n.parent.color = Color.BLACK; 281 | } else 282 | deleteCase5(n); 283 | } 284 | 285 | private void deleteCase5(Node n) { 286 | if (n == n.parent.left && 287 | nodeColor(n.sibling()) == Color.BLACK && 288 | nodeColor(n.sibling().left) == Color.RED && 289 | nodeColor(n.sibling().right) == Color.BLACK) { 290 | n.sibling().color = Color.RED; 291 | n.sibling().left.color = Color.BLACK; 292 | rotateRight(n.sibling()); 293 | } else if (n == n.parent.right && 294 | nodeColor(n.sibling()) == Color.BLACK && 295 | nodeColor(n.sibling().right) == Color.RED && 296 | nodeColor(n.sibling().left) == Color.BLACK) { 297 | n.sibling().color = Color.RED; 298 | n.sibling().right.color = Color.BLACK; 299 | rotateLeft(n.sibling()); 300 | } 301 | deleteCase6(n); 302 | } 303 | 304 | private void deleteCase6(Node n) { 305 | n.sibling().color = nodeColor(n.parent); 306 | n.parent.color = Color.BLACK; 307 | if (n == n.parent.left) { 308 | assert nodeColor(n.sibling().right) == Color.RED; 309 | n.sibling().right.color = Color.BLACK; 310 | rotateLeft(n.parent); 311 | } else { 312 | assert nodeColor(n.sibling().left) == Color.RED; 313 | n.sibling().left.color = Color.BLACK; 314 | rotateRight(n.parent); 315 | } 316 | } 317 | 318 | public void print() { 319 | printHelper(root, 0); 320 | } 321 | 322 | private static void printHelper(Node n, int indent) { 323 | if (n == null) { 324 | System.out.print(""); 325 | return; 326 | } 327 | if (n.right != null) { 328 | printHelper(n.right, indent + INDENT_STEP); 329 | } 330 | for (int i = 0; i < indent; i++) 331 | System.out.print(" "); 332 | if (n.color == Color.BLACK) 333 | System.out.println(n.key); 334 | else 335 | System.out.println("<" + n.key + ">"); 336 | if (n.left != null) { 337 | printHelper(n.left, indent + INDENT_STEP); 338 | } 339 | } 340 | 341 | public int count() { 342 | final int[] counter = new int[1]; 343 | counter[0] = 0; 344 | 345 | Visitor visitor = new Visitor() { 346 | public void visit(Node node) { 347 | counter[0] = counter[0] + 1; 348 | if (node.left != null) { 349 | visit(node.left); 350 | } 351 | if (node.right != null) { 352 | visit(node.right); 353 | } 354 | } 355 | }; 356 | visitor.visit(root); 357 | return counter[0]; 358 | } 359 | 360 | public static void main(String[] args) { 361 | RBTree t = new RBTree(); 362 | t.print(); 363 | 364 | java.util.Random gen = new java.util.Random(); 365 | 366 | int dups = 0; 367 | for (int i = 0; i < 5000; i++) { 368 | int x = gen.nextInt(10000); 369 | long y = gen.nextInt(10000); 370 | 371 | // t.print(); 372 | System.out.print("" + x + " -> " + y + ","); 373 | System.out.println(); 374 | if(t.lookup(x) != null){ 375 | dups++; 376 | } 377 | t.insert(x, y); 378 | assert t.lookup(x)[0] == y; 379 | } 380 | System.out.print(String.format("Expected:%d, Actual: %d ", 5000 -dups, t.count())); 381 | for (int i = 0; i < 60000; i++) { 382 | int x = gen.nextInt(10000); 383 | 384 | // t.print(); 385 | System.out.print("Deleting key " + x); 386 | if (t.lookup(x) != null) { 387 | t.delete(x, t.lookup(x)[0]); 388 | } 389 | } 390 | } 391 | 392 | private int rehash(int h) { 393 | h ^= (h >>> 20) ^ (h >>> 12); 394 | return h ^ (h >>> 7) ^ (h >>> 4); 395 | } 396 | 397 | public void writeExternal(ObjectOutput objectOutput) throws IOException { 398 | root.writeExternal(objectOutput); 399 | } 400 | 401 | public void readExternal(ObjectInput objectInput) throws IOException, ClassNotFoundException { 402 | Node node = new Node(); 403 | node.readExternal(objectInput); 404 | root = node; 405 | } 406 | 407 | public interface Visitor { 408 | void visit(Node node); 409 | } 410 | } 411 | -------------------------------------------------------------------------------- /src/com/alok/diskmap/Record.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap; 18 | 19 | import java.io.DataInput; 20 | import java.io.DataOutput; 21 | import java.io.IOException; 22 | import java.nio.ByteBuffer; 23 | import java.nio.channels.ByteChannel; 24 | import java.nio.channels.FileChannel; 25 | import java.util.logging.Level; 26 | import java.util.logging.Logger; 27 | 28 | public class Record implements Comparable{ 29 | private static final ConversionUtils util = new ConversionUtils(); 30 | private static final Logger logger = Logger.getLogger(Record.class.getName()); 31 | 32 | private final boolean DEBUG = false; 33 | public static final int ACTIVE = 1; 34 | public static final int DELETED = 2; 35 | public static final int EMPTY = 4; 36 | 37 | private int flag; 38 | private int hash; 39 | private int keySize; 40 | private byte[] key; 41 | private int valueSize; 42 | private byte[] value; 43 | private long location = -1; 44 | 45 | public Record() { 46 | } 47 | 48 | public Record(byte[] key, byte[] value, int flag, int hash, long location) { 49 | this.flag = flag; 50 | this.hash = hash; 51 | this.keySize = key.length; 52 | this.key = key; 53 | this.valueSize = value.length; 54 | this.value = value; 55 | this.location = location; 56 | } 57 | 58 | public Record(Record r, long newLocation) { 59 | this(r.getKey(), r.getValue(), r.getFlag(), r.getHash(), newLocation); 60 | } 61 | 62 | public void setFlag(int flag) { 63 | this.flag = flag; 64 | } 65 | 66 | public void write(DataOutput index) throws IOException { 67 | writeIndex(index); 68 | writeDate(index); 69 | } 70 | 71 | public void read(FileChannel index, long location) throws IOException { 72 | try { 73 | doMmapRead(index, location); 74 | } catch (IOException e) { 75 | System.gc(); 76 | System.runFinalization(); 77 | doMmapRead(index, location); 78 | } 79 | } 80 | 81 | private void doMmapRead(FileChannel index, long location) throws IOException { 82 | ByteBuffer metaBuffer; 83 | metaBuffer = index.map(FileChannel.MapMode.READ_ONLY, location, 29); 84 | readIndex(metaBuffer); 85 | readHeader(metaBuffer); 86 | metaBuffer = null; 87 | ByteBuffer dataBuffer = index.map(FileChannel.MapMode.READ_ONLY, location + 29, keySize + valueSize + 2); 88 | readData(dataBuffer); 89 | dataBuffer = null; 90 | } 91 | 92 | public void read(DataInput index) throws IOException { 93 | byte[] bytes = new byte[29]; 94 | index.readFully(bytes); 95 | ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); 96 | readIndex(byteBuffer); 97 | readHeader(byteBuffer); 98 | bytes = new byte[keySize + valueSize + 2]; 99 | index.readFully(bytes); 100 | byteBuffer = ByteBuffer.wrap(bytes); 101 | readData(byteBuffer); 102 | } 103 | 104 | public void writeIndex(DataOutput index) throws IOException { 105 | index.write(util.intToBytes(flag)); 106 | index.write(0); 107 | index.write(util.intToBytes(hash)); 108 | index.write(0); 109 | index.write(util.longToBytes(location)); 110 | index.write(0); 111 | } 112 | 113 | //Reads 4 + 1 + 4 + 1 + 8 + 1 = 19bytes 114 | private void readIndex(ByteBuffer index) throws IOException { 115 | setFlag(readInt(index)); 116 | index.get(); 117 | this.hash = readInt(index); 118 | index.get(); 119 | this.location = readLong(index); 120 | index.get(); 121 | } 122 | 123 | //Reads 4 + 1 + 4 + 1 = 10 bytes 124 | private void readHeader(ByteBuffer index) throws IOException { 125 | this.keySize = readInt(index); 126 | index.get(); 127 | this.valueSize = readInt(index); 128 | index.get(); 129 | if(DEBUG){ 130 | logger.log(Level.SEVERE, String.format("keySize[%d], valueSize[%d]", keySize, valueSize)); 131 | if(keySize > 1000){ 132 | logger.log(Level.SEVERE, "Large key"); 133 | } 134 | } 135 | } 136 | 137 | //Reads keySize + 1+ valueSize + 1 bytes 138 | private void readData(ByteBuffer index) throws IOException { 139 | this.key = new byte[keySize]; 140 | this.value = new byte[valueSize]; 141 | index.get(key); 142 | index.get(); 143 | index.get(value); 144 | index.get(); 145 | } 146 | 147 | public void writeDate(DataOutput index) throws IOException { 148 | index.write(util.intToBytes(keySize)); 149 | index.write(0); 150 | index.write(util.intToBytes(valueSize)); 151 | index.write(0); 152 | index.write(key); 153 | index.write(0); 154 | index.write(value); 155 | index.write(0); 156 | } 157 | 158 | private int readInt(ByteBuffer file) throws IOException { 159 | byte[] buffer = new byte[4]; 160 | file.get(buffer); 161 | return util.byteToInt(buffer); 162 | } 163 | 164 | private long readLong(ByteBuffer file) throws IOException { 165 | byte[] buffer = new byte[8]; 166 | file.get(buffer); 167 | return util.byteToLong(buffer); 168 | } 169 | 170 | public boolean equals(Object other) { 171 | if (other != null && !(other instanceof Record)) { 172 | return false; 173 | } 174 | Record that = (Record) other; 175 | if (this.flag != that.flag) { 176 | return false; 177 | } 178 | if (this.hash != that.hash) { 179 | return false; 180 | } 181 | if (this.keySize != that.keySize) { 182 | return false; 183 | } 184 | if (this.valueSize != that.valueSize) { 185 | return false; 186 | } 187 | 188 | if (this.location != that.location) { 189 | return false; 190 | } 191 | 192 | if (this.key.length != that.key.length) { 193 | return false; 194 | } 195 | for (int i = 0; i < this.key.length; i++) { 196 | if (this.key[i] != that.key[i]) { 197 | return false; 198 | } 199 | } 200 | 201 | if (this.value.length != that.value.length) { 202 | return false; 203 | } 204 | for (int i = 0; i < this.value.length; i++) { 205 | if (this.value[i] != that.value[i]) { 206 | return false; 207 | } 208 | } 209 | return true; 210 | } 211 | 212 | public int getHash() { 213 | return hash; 214 | } 215 | 216 | public int getFlag() { 217 | return flag; 218 | } 219 | 220 | public byte[] getKey() { 221 | return key; 222 | } 223 | 224 | public byte[] getValue() { 225 | return value; 226 | } 227 | 228 | public long getLocation() { 229 | return location; 230 | } 231 | 232 | public int size() { 233 | return keySize + valueSize + (4 * 4) + 8;// int fields + long fields 234 | } 235 | 236 | @Override 237 | public String toString() { 238 | return "Record{" + 239 | "key=" + util.deserialize(key) + 240 | "value =" + util.deserialize(value) + 241 | "keySize =" + keySize+ 242 | "valueSize =" + valueSize + 243 | ", hash=" + hash + 244 | ", flag=" + flag + 245 | ", location=" + location + 246 | '}'; 247 | } 248 | 249 | public void setLocation(long location) { 250 | this.location = location; 251 | } 252 | 253 | @Override 254 | public int compareTo(Record o) { 255 | return location > o.location ? 1 : (location < o.location ? -1 : 0); 256 | } 257 | } 258 | -------------------------------------------------------------------------------- /src/com/alok/diskmap/ZipMap.java: -------------------------------------------------------------------------------- 1 | package com.alok.diskmap; 2 | 3 | import java.util.*; 4 | import java.util.concurrent.locks.Lock; 5 | import java.util.concurrent.locks.ReadWriteLock; 6 | import java.util.concurrent.locks.ReentrantReadWriteLock; 7 | 8 | public class ZipMap implements Map{ 9 | private byte[] data; 10 | private int length; 11 | private int dataSize; 12 | private int items; 13 | private ConversionUtils utils = ConversionUtils.instance; 14 | ReadWriteLock rwl = new ReentrantReadWriteLock(); 15 | private static final int KEY_SIZE_LEN = 2; 16 | private static final int VALUE_SIZE_LEN = 2; 17 | private static final int SIZE_LEN = KEY_SIZE_LEN + VALUE_SIZE_LEN; 18 | private static final int CRC_LEN = 2; 19 | 20 | public ZipMap(){ 21 | init(); 22 | } 23 | 24 | private void init() { 25 | this.data = new byte[1024]; 26 | this.length = 0; 27 | this.items = 0; 28 | this.dataSize = 0; 29 | } 30 | 31 | public int size() { 32 | return items; 33 | } 34 | 35 | public boolean isEmpty() { 36 | return items == 0; 37 | } 38 | 39 | public boolean containsKey(Object o) { 40 | return get(o) != null; 41 | } 42 | 43 | public boolean containsValue(Object o) { 44 | Lock lock = rwl.readLock(); 45 | lock.lock(); 46 | if(length > 0){ 47 | int offset = 0; 48 | byte[] value = (byte[])o; 49 | while(offset < length){ 50 | short keySize = utils.byteToShort(data, offset); 51 | short valueSize = utils.byteToShort(data, offset + KEY_SIZE_LEN); 52 | if(valueSize == value.length){ 53 | if(areEqual(value, data, offset + SIZE_LEN + CRC_LEN)){ 54 | return true; 55 | } 56 | } 57 | offset = offset + keySize + valueSize + SIZE_LEN + CRC_LEN; 58 | } 59 | } 60 | lock.unlock(); 61 | return false; 62 | } 63 | 64 | public byte[] remove(Object o) { 65 | byte[] value = null; 66 | Lock lock = rwl.writeLock(); 67 | lock.lock(); 68 | if(length > 0){ 69 | int offset = 0; 70 | byte[] keyBytes = ((String)o).getBytes(); 71 | short keyCrc = utils.crc16(keyBytes); 72 | while((offset + SIZE_LEN + CRC_LEN) < (length)){ 73 | short keySize = utils.byteToShort(data, offset); 74 | short valueSize = utils.byteToShort(data, offset + KEY_SIZE_LEN); 75 | if(keySize == keyBytes.length){ 76 | short crc = utils.byteToShort(data, offset + SIZE_LEN); 77 | if(keyCrc == crc && areEqual(keyBytes, data, offset + SIZE_LEN + CRC_LEN)){ 78 | value = new byte[valueSize]; 79 | System.arraycopy(data, offset + SIZE_LEN + CRC_LEN, value, 0, valueSize); 80 | for(int i = 0; i < (valueSize+keySize+CRC_LEN); i++){ 81 | data[offset + SIZE_LEN + i] = 0; 82 | } 83 | items--; 84 | dataSize = dataSize - keySize - valueSize - SIZE_LEN - CRC_LEN; 85 | } 86 | } 87 | offset = offset + keySize + valueSize + SIZE_LEN + CRC_LEN; 88 | } 89 | } 90 | lock.unlock(); 91 | return value; 92 | } 93 | 94 | private boolean areEqual(byte[] source, byte[] target, int offset) { 95 | int i = 0; 96 | for (byte b : source) { 97 | if(b != target[offset + i]){ 98 | return false; 99 | } 100 | i++; 101 | } 102 | return true; 103 | } 104 | 105 | public byte[] put(String key, byte[] bytes) { 106 | byte[] value = null; 107 | Lock lock = rwl.writeLock(); 108 | lock.lock(); 109 | byte[] currentValue = _get(key); 110 | if(currentValue == null){ 111 | byte[] keyBytes = key.getBytes(); 112 | byte[] crcBytes = utils.shortToBytes(utils.crc16(keyBytes)); 113 | if((bytes.length + keyBytes.length + CRC_LEN + SIZE_LEN) > (data.length - length)){ 114 | int increase = bytes.length + keyBytes.length + SIZE_LEN + CRC_LEN - (data.length - length); 115 | byte[] newData = new byte[data.length + increase]; 116 | System.arraycopy(data, 0, newData, 0, data.length); 117 | data = newData; 118 | } 119 | byte[] keyLenBytes = utils.shortToBytes(key.getBytes().length); 120 | byte[] valueLenBytes = utils.shortToBytes(bytes.length); 121 | System.arraycopy(keyLenBytes, 0, data, length, keyLenBytes.length); 122 | System.arraycopy(valueLenBytes, 0, data, length + KEY_SIZE_LEN, valueLenBytes.length); 123 | System.arraycopy(crcBytes, 0, data, length + SIZE_LEN, CRC_LEN); 124 | System.arraycopy(keyBytes, 0, data, length + SIZE_LEN + CRC_LEN, keyBytes.length); 125 | System.arraycopy(bytes, 0, data, length + SIZE_LEN + CRC_LEN + keyBytes.length, bytes.length); 126 | length += keyBytes.length + bytes.length + SIZE_LEN + CRC_LEN; 127 | items++; 128 | dataSize += keyBytes.length + bytes.length + SIZE_LEN + CRC_LEN; 129 | return null; 130 | }else{ 131 | remove(key); 132 | value = put(key, bytes); 133 | } 134 | lock.unlock(); 135 | return value; 136 | } 137 | 138 | public byte[] get(Object o) { 139 | Lock lock = rwl.readLock(); 140 | lock.lock(); 141 | byte[] value = _get(o); 142 | lock.unlock(); 143 | return value; 144 | } 145 | public byte[] _get(Object o) { 146 | if(length > 0){ 147 | int offset = 0; 148 | byte[] keyBytes = ((String)o).getBytes(); 149 | short keyCrc = utils.crc16(keyBytes); 150 | while((offset + SIZE_LEN + CRC_LEN) < length){ 151 | short keySize = utils.byteToShort(data, offset); 152 | short valueSize = utils.byteToShort(data, offset + KEY_SIZE_LEN); 153 | if(keySize < 0 || valueSize < 0){ 154 | System.out.println("Error"); 155 | } 156 | if(keySize == keyBytes.length){ 157 | short crc = utils.byteToShort(data, offset + SIZE_LEN); 158 | if(crc == keyCrc && areEqual(keyBytes, data, offset + SIZE_LEN + CRC_LEN)){ 159 | byte[] value = new byte[valueSize]; 160 | System.arraycopy(data, offset + SIZE_LEN + CRC_LEN + keySize , value, 0, valueSize); 161 | return value; 162 | } 163 | } 164 | offset = offset + keySize + valueSize + SIZE_LEN + CRC_LEN; 165 | } 166 | } 167 | return null; 168 | } 169 | 170 | public void putAll(Map map) { 171 | for (String key : map.keySet()) { 172 | put(key, map.get(key)); 173 | } 174 | } 175 | 176 | public void clear() { 177 | Lock lock = rwl.writeLock(); 178 | lock.lock(); 179 | init(); 180 | lock.unlock(); 181 | } 182 | 183 | public Set keySet() { 184 | Set keys = new HashSet(); 185 | Lock lock = rwl.readLock(); 186 | lock.lock(); 187 | if(length > 0){ 188 | int offset = 0; 189 | while(offset < length){ 190 | short keySize = utils.byteToShort(data, offset); 191 | short valueSize = utils.byteToShort(data, offset + KEY_SIZE_LEN); 192 | String key = new String(data, offset + SIZE_LEN + CRC_LEN, keySize); 193 | keys.add(key); 194 | offset = offset + keySize + valueSize + SIZE_LEN + CRC_LEN; 195 | } 196 | } 197 | lock.unlock(); 198 | return keys; 199 | } 200 | 201 | public Collection values() { 202 | Lock lock = rwl.readLock(); 203 | lock.lock(); 204 | List values = new ArrayList(); 205 | if(length > 0){ 206 | int offset = 0; 207 | while(offset < length){ 208 | short keySize = utils.byteToShort(data, offset); 209 | short valueSize = utils.byteToShort(data, offset + KEY_SIZE_LEN); 210 | byte[] value = new byte[valueSize]; 211 | System.arraycopy(data, offset + SIZE_LEN + CRC_LEN + keySize, value, 0, valueSize); 212 | values.add(value); 213 | offset = offset + keySize + valueSize + SIZE_LEN + CRC_LEN; 214 | } 215 | } 216 | lock.unlock(); 217 | return values; 218 | } 219 | 220 | public Set> entrySet() { 221 | Lock lock = rwl.readLock(); 222 | lock.lock(); 223 | Set> entries = new HashSet>(); 224 | if(length > 0){ 225 | int offset = 0; 226 | while(offset < length){ 227 | short keySize = utils.byteToShort(data, offset); 228 | short valueSize = utils.byteToShort(data, offset + KEY_SIZE_LEN); 229 | final String key = new String(data, offset + SIZE_LEN + CRC_LEN, keySize); 230 | final byte[] value = new byte[valueSize]; 231 | System.arraycopy(data, offset + SIZE_LEN + CRC_LEN + keySize, value, 0, valueSize); 232 | offset = offset + keySize + valueSize + SIZE_LEN + CRC_LEN; 233 | entries.add(new Entry(){ 234 | 235 | public Object getKey() { 236 | return key; 237 | } 238 | 239 | public Object getValue() { 240 | return value; 241 | } 242 | 243 | public Object setValue(Object o) { 244 | throw new UnsupportedOperationException("setValue not supported"); 245 | } 246 | }); 247 | } 248 | } 249 | lock.unlock(); 250 | return entries; 251 | } 252 | } 253 | -------------------------------------------------------------------------------- /src/com/alok/diskmap/io/BaseDiskIO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.io; 18 | 19 | import com.alok.diskmap.Configuration; 20 | import com.alok.diskmap.Record; 21 | 22 | import java.io.File; 23 | import java.io.IOException; 24 | import java.io.RandomAccessFile; 25 | import java.util.Arrays; 26 | import java.util.Iterator; 27 | import java.util.logging.Level; 28 | import java.util.logging.Logger; 29 | 30 | public abstract class BaseDiskIO implements DiskIO { 31 | private static final Logger logger = Logger.getLogger(BlockingDiskIO.class.getName()); 32 | protected File file; 33 | private RandomAccessFile writer; 34 | private RandomAccessFile reader; 35 | private long lastFlush; 36 | protected Configuration config; 37 | private static final boolean DEBUG = false; 38 | 39 | public BaseDiskIO(Configuration config, File f){ 40 | try { 41 | this.config = config; 42 | if(f == null){ 43 | this.file = new File(config.getDataFileName("dat")); 44 | } 45 | if (!this.file.exists()) { 46 | boolean created = this.file.createNewFile(); 47 | if(!created){ 48 | throw new RuntimeException(String.format("Unable to create file: %s", this.file.getAbsolutePath())); 49 | } 50 | } 51 | createFileHandlers(); 52 | }catch(Exception e){ 53 | throw newRuntimeException(e); 54 | } 55 | } 56 | 57 | protected void createFileHandlers() { 58 | try{ 59 | this.setReader(new RandomAccessFile(this.file, "r")); 60 | this.setWriter(new RandomAccessFile(this.file, "rw")); 61 | this.getWriter().seek(reader().length()); 62 | }catch(Exception e){ 63 | close(getReader()); 64 | close(getWriter()); 65 | } 66 | } 67 | 68 | @Override 69 | public Iterator iterator() { 70 | final RandomAccessFile rc; 71 | try { 72 | rc = new RandomAccessFile(file, "r"); 73 | if (rc.length() > 0) { 74 | rc.seek(0); 75 | } 76 | } catch (Exception e) { 77 | throw newRuntimeException(e); 78 | } 79 | 80 | return new Iterator(){ 81 | @Override 82 | public boolean hasNext() { 83 | try { 84 | if(rc.getFilePointer() < rc.length()){ 85 | return true; 86 | } 87 | close(rc); 88 | return false; 89 | } catch (IOException e) { 90 | close(rc); 91 | throw newRuntimeException(e); 92 | } 93 | } 94 | 95 | @Override 96 | public Record next() { 97 | try { 98 | Record r = new Record(); 99 | r.read(rc); 100 | return r; 101 | } catch (IOException e) { 102 | close(rc); 103 | throw newRuntimeException(e); 104 | } 105 | } 106 | 107 | @Override 108 | public void remove() { 109 | throw new UnsupportedOperationException("Remove is not supported"); 110 | } 111 | }; 112 | } 113 | 114 | private void close(RandomAccessFile rc) { 115 | try {rc.close();} catch (IOException ioe) {logger.log(Level.SEVERE, ioe.getMessage(), ioe);} 116 | } 117 | 118 | protected RuntimeException newRuntimeException(Exception e) throws RuntimeException{ 119 | logger.log(Level.SEVERE, e.getMessage(), e); 120 | throw new RuntimeException(e); 121 | } 122 | 123 | @Override 124 | public void close() { 125 | close(this.getWriter()); 126 | close(this.getReader()); 127 | this.file = null; 128 | } 129 | 130 | public abstract Record lookup(long location); 131 | 132 | public Record doLookup(long location) { 133 | try { 134 | Record r = new Record(); 135 | RandomAccessFile reader = reader(); 136 | synchronized (reader){ 137 | reader.seek(location); 138 | r.read(reader); 139 | // r.read(reader.getChannel(), location); 140 | } 141 | return r; 142 | } catch (IOException e) { 143 | logger.log(Level.SEVERE, String.format("lookup(%d) failed", location)); 144 | throw newRuntimeException(e); 145 | } 146 | } 147 | 148 | @Override 149 | public abstract long write(Record r); 150 | 151 | protected long doWrite(Record r, RandomAccessFile writer) throws IOException{ 152 | long location = writer.getFilePointer(); 153 | Record newRecord = new Record(r, location); 154 | newRecord.write(writer); 155 | return location; 156 | } 157 | 158 | protected void doFlush() { 159 | try { 160 | writer().getChannel().force(false); 161 | this.lastFlush = System.currentTimeMillis(); 162 | } catch (Exception e) { 163 | throw new RuntimeException(e); 164 | } 165 | } 166 | 167 | protected final RandomAccessFile writer() { 168 | return getWriter(); 169 | } 170 | 171 | protected final RandomAccessFile reader() { 172 | return getReader(); 173 | } 174 | 175 | protected Configuration getConfig() { 176 | return config; 177 | } 178 | 179 | @Override 180 | public abstract void vacuum(RecordFilter filter) throws Exception; 181 | 182 | @Override 183 | public void clear() { 184 | closeFileHandlers(); 185 | boolean b = new File(config.getDataFileName("dat")).delete(); 186 | if(b){ 187 | try { 188 | boolean created = new File(config.getDataFileName("dat")).createNewFile(); 189 | if(created){ 190 | createFileHandlers(); 191 | return; 192 | } 193 | } catch (IOException e) { 194 | throw new RuntimeException("Unable to clear file: " + config.getDataFileName("dat"), e); 195 | } 196 | } 197 | throw new RuntimeException("Unable to clear file: " + config.getDataFileName("dat")); 198 | } 199 | 200 | public void doVacuum(RecordFilter filter) throws Exception { 201 | doFlush(); 202 | closeFileHandlers(); 203 | File newFile = new File(config.getDataFileName("tmp")); 204 | RandomAccessFile newWriter = new RandomAccessFile(newFile, "rw"); 205 | for(Record r : this){ 206 | if(filter.accept(r)){ 207 | long location = doWrite(r, newWriter); 208 | filter.update(r, location); 209 | } 210 | } 211 | newWriter.close(); 212 | if(this.file.renameTo(new File(config.getDataFileName("bak")))){ 213 | if(newFile.renameTo(new File(config.getDataFileName("dat")))){ 214 | this.file = new File(config.getDataFileName("dat")); 215 | createFileHandlers(); 216 | new File(config.getDataFileName("bak")).delete(); 217 | }else{ 218 | throw new RuntimeException("Unable to vacuum the data file."); 219 | } 220 | }else{ 221 | throw new RuntimeException("Unable to vacuum the data file."); 222 | } 223 | } 224 | 225 | @Override 226 | public long size(){ 227 | try { 228 | return writer().getFilePointer(); 229 | } catch (IOException e) { 230 | logger.log(Level.SEVERE, e.getMessage(), e); 231 | } 232 | return 0; 233 | } 234 | 235 | private void closeFileHandlers() { 236 | close(getReader()); 237 | close(getWriter()); 238 | } 239 | 240 | @Override 241 | public abstract void update(Record...records); 242 | 243 | @Override 244 | public abstract void update(Record record); 245 | 246 | public void doUpdate(Record record) throws IOException { 247 | long currentLocation = writer().getFilePointer(); 248 | writer().seek(record.getLocation()); 249 | record.write(writer()); 250 | writer().seek(currentLocation); 251 | } 252 | 253 | 254 | public void doUpdate(Record...records) throws IOException { 255 | long currentLocation = writer().getFilePointer(); 256 | Arrays.sort(records); 257 | for (Record record : records) { 258 | writer().seek(record.getLocation()); 259 | record.write(writer()); 260 | } 261 | writer().seek(currentLocation); 262 | } 263 | 264 | private RandomAccessFile getWriter() { 265 | return writer; 266 | } 267 | 268 | public void setWriter(RandomAccessFile writer) { 269 | this.writer = writer; 270 | } 271 | 272 | private RandomAccessFile getReader() { 273 | return reader; 274 | } 275 | 276 | public void setReader(RandomAccessFile reader) { 277 | this.reader = reader; 278 | } 279 | 280 | protected long getLastFlush() { 281 | return lastFlush; 282 | } 283 | 284 | public boolean isDebug(){ 285 | return DEBUG; 286 | } 287 | } 288 | -------------------------------------------------------------------------------- /src/com/alok/diskmap/io/BlockingDiskIO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.io; 18 | 19 | import com.alok.diskmap.Configuration; 20 | import com.alok.diskmap.Record; 21 | 22 | import java.io.File; 23 | import java.io.IOException; 24 | import java.util.logging.Level; 25 | import java.util.logging.Logger; 26 | 27 | public class BlockingDiskIO extends BaseDiskIO { 28 | 29 | private static final Logger logger = Logger.getLogger(BlockingDiskIO.class.getName()); 30 | 31 | public BlockingDiskIO(Configuration config, String file){ 32 | super(config, new File(file)); 33 | } 34 | public BlockingDiskIO(Configuration config){ 35 | super(config, null); 36 | } 37 | public BlockingDiskIO(Configuration config, File f){ 38 | super(config, f); 39 | } 40 | 41 | @Override 42 | public Record lookup(long location) { 43 | return doLookup(location); 44 | } 45 | 46 | @Override 47 | public long write(Record r) { 48 | try { 49 | return doWrite(r, writer()); 50 | } catch (IOException e) { 51 | throw newRuntimeException(e); 52 | } 53 | } 54 | 55 | 56 | private void flush() { 57 | if(System.currentTimeMillis() - this.getLastFlush() >= getConfig().getFlushInterval()){ 58 | doFlush(); 59 | } 60 | } 61 | 62 | @Override 63 | public void vacuum(RecordFilter filter) throws Exception { 64 | doVacuum(filter); 65 | } 66 | 67 | @Override 68 | public void update(Record record) { 69 | try { 70 | doUpdate(record); 71 | flush(); 72 | } catch (IOException e) { 73 | logger.log(Level.SEVERE, e.getMessage(), e); 74 | } 75 | } 76 | 77 | @Override 78 | public void update(Record...records) { 79 | try { 80 | doUpdate(records); 81 | flush(); 82 | } catch (IOException e) { 83 | logger.log(Level.SEVERE, e.getMessage(), e); 84 | } 85 | } 86 | 87 | 88 | } 89 | -------------------------------------------------------------------------------- /src/com/alok/diskmap/io/DiskIO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.io; 18 | 19 | import com.alok.diskmap.Record; 20 | 21 | public interface DiskIO extends Iterable { 22 | 23 | Record lookup(long location); 24 | 25 | long write(Record r); 26 | 27 | void update(Record r); 28 | 29 | void update(Record...rs); 30 | 31 | long size(); 32 | 33 | void close(); 34 | 35 | void vacuum(RecordFilter filter) throws Exception; 36 | 37 | void clear(); 38 | 39 | public interface RecordFilter{ 40 | public boolean accept(Record r); 41 | 42 | void update(Record r, long newLocation); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/com/alok/diskmap/io/NonBlockingDiskIO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.io; 18 | 19 | import com.alok.diskmap.Configuration; 20 | import com.alok.diskmap.Record; 21 | 22 | import java.io.IOException; 23 | import java.util.ArrayList; 24 | import java.util.Collections; 25 | import java.util.List; 26 | import java.util.concurrent.*; 27 | import java.util.concurrent.atomic.AtomicBoolean; 28 | 29 | public class NonBlockingDiskIO extends BaseDiskIO implements DiskIO{ 30 | private final BlockingQueue readQueue; 31 | private final Thread readerThread; 32 | private final ReaderTask readerTask; 33 | 34 | public NonBlockingDiskIO(Configuration config){ 35 | super(config, null); 36 | this.readQueue = new LinkedBlockingQueue(); 37 | this.readerTask = new ReaderTask(readQueue); 38 | this.readerThread = new Thread(readerTask, "ReaderThread-" + config.getNumber()); 39 | this.readerThread.start(); 40 | } 41 | 42 | @Override 43 | public Record lookup(long location) { 44 | ReadFuture task = new ReadFuture(location); 45 | try { 46 | this.readQueue.put(task); 47 | return task.get(); 48 | } catch (InterruptedException e) { 49 | throw newRuntimeException(e); 50 | } catch (ExecutionException e) { 51 | throw newRuntimeException(e); 52 | } 53 | } 54 | 55 | @Override 56 | public long write(Record r) { 57 | try { 58 | return doWrite(r, writer()); 59 | } catch (IOException e) { 60 | throw newRuntimeException(e); 61 | } 62 | } 63 | 64 | @Override 65 | public void vacuum(RecordFilter filter) throws Exception { 66 | doVacuum(filter); 67 | } 68 | 69 | @Override 70 | public void update(Record record) { 71 | try { 72 | doUpdate(record); 73 | } catch (IOException e) { 74 | throw newRuntimeException(e); 75 | } 76 | } 77 | 78 | @Override 79 | public void update(Record...records) { 80 | try { 81 | doUpdate(records); 82 | } catch (IOException e) { 83 | throw newRuntimeException(e); 84 | } 85 | } 86 | 87 | @Override 88 | public void close(){ 89 | readerTask.stop(); 90 | super.close(); 91 | } 92 | 93 | public class ReaderTask implements Runnable{ 94 | private BlockingQueue readQueue; 95 | private AtomicBoolean shouldRun = new AtomicBoolean(true); 96 | 97 | public ReaderTask(BlockingQueue readQueue){ 98 | this.readQueue = readQueue; 99 | } 100 | @Override 101 | public void run() { 102 | while(shouldRun()){ 103 | try { 104 | List readFutures = new ArrayList(); 105 | NonBlockingDiskIO.ReadFuture item = readQueue.poll(1000, TimeUnit.MILLISECONDS); 106 | if(item == null){ 107 | continue; 108 | } 109 | readFutures.add(item); 110 | readQueue.drainTo(readFutures); 111 | Collections.sort(readFutures); 112 | for (ReadFuture future : readFutures) { 113 | Record record = doLookup(future.getLocation()); 114 | future.complete(record); 115 | } 116 | } catch (InterruptedException e) { 117 | e.printStackTrace(); 118 | } 119 | } 120 | } 121 | 122 | private boolean shouldRun() { 123 | return shouldRun.get(); 124 | } 125 | 126 | public void stop(){ 127 | this.shouldRun.set(false); 128 | } 129 | } 130 | 131 | public class ReadFuture implements Future, Comparable{ 132 | private final long location; 133 | private Record r; 134 | private final AtomicBoolean isDone = new AtomicBoolean(false); 135 | public ReadFuture(long location){ 136 | this.location = location; 137 | } 138 | @Override 139 | public boolean cancel(boolean mayInterruptIfRunning) { 140 | return false; 141 | } 142 | 143 | @Override 144 | public boolean isCancelled() { 145 | return false; 146 | } 147 | 148 | @Override 149 | public boolean isDone() { 150 | return isDone.get(); 151 | } 152 | 153 | @Override 154 | public Record get() throws InterruptedException, ExecutionException { 155 | synchronized (isDone){ 156 | while(!isDone.get()){ 157 | isDone.wait(); 158 | } 159 | if(isDone()){ 160 | return r; 161 | } 162 | } 163 | throw new ExecutionException(new RuntimeException("Did not complete the lookup")); 164 | } 165 | 166 | public void complete(Record r){ 167 | this.r = r; 168 | synchronized (isDone){ 169 | isDone.set(true); 170 | isDone.notifyAll(); 171 | } 172 | } 173 | 174 | @Override 175 | public Record get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { 176 | return get(); 177 | } 178 | 179 | public long getLocation() { 180 | return location; 181 | } 182 | 183 | @Override 184 | public int compareTo(ReadFuture o) { 185 | return this.location > o.location ? 1 : (this.location < o.location ? -1 : 0); 186 | } 187 | } 188 | } 189 | 190 | -------------------------------------------------------------------------------- /src/com/alok/diskmap/utils/DefaultObjectConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.utils; 18 | 19 | import java.io.*; 20 | 21 | public class DefaultObjectConverter implements ObjectConverter { 22 | @Override 23 | public byte[] serialize(Serializable object) throws Exception { 24 | ByteArrayOutputStream bout = new ByteArrayOutputStream(); 25 | new ObjectOutputStream(bout).writeObject(object); 26 | bout.flush(); 27 | return bout.toByteArray(); 28 | } 29 | 30 | @Override 31 | public T deserialize(byte[] buffer) { 32 | try { 33 | ObjectInputStream ios = new ObjectInputStream(new ByteArrayInputStream(buffer)); 34 | return (T) ios.readObject(); 35 | } catch (IOException e) { 36 | throw new RuntimeException(e); 37 | } catch (ClassNotFoundException e) { 38 | throw new RuntimeException(e); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/com/alok/diskmap/utils/Hessian2ObjectConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.utils; 18 | 19 | import com.caucho.hessian.io.*; 20 | 21 | import java.io.ByteArrayInputStream; 22 | import java.io.ByteArrayOutputStream; 23 | import java.io.Serializable; 24 | import java.math.BigDecimal; 25 | 26 | public class Hessian2ObjectConverter implements ObjectConverter { 27 | private Hessian2Output os; 28 | private SerializerFactory factory; 29 | 30 | public Hessian2ObjectConverter() { 31 | this.os = new Hessian2Output(null); 32 | this.factory = new SerializerFactory(); 33 | factory.addFactory(new AbstractSerializerFactory(){ 34 | @Override 35 | public Serializer getSerializer(Class cl) throws HessianProtocolException { 36 | if(cl.isAssignableFrom(BigDecimal.class)){ 37 | return StringValueSerializer.SER; 38 | } 39 | return null; 40 | } 41 | 42 | @Override 43 | public Deserializer getDeserializer(Class cl) throws HessianProtocolException { 44 | if(cl.isAssignableFrom(BigDecimal.class)){ 45 | return new StringValueDeserializer(BigDecimal.class); 46 | } 47 | return null; 48 | } 49 | }); 50 | this.os.setSerializerFactory(factory); 51 | 52 | 53 | } 54 | 55 | @Override 56 | public byte[] serialize(Serializable object) throws Exception { 57 | ByteArrayOutputStream buffer = new ByteArrayOutputStream(); 58 | synchronized (os) { 59 | os.init(buffer); 60 | os.writeObject(object); 61 | os.close(); 62 | } 63 | return buffer.toByteArray(); 64 | } 65 | 66 | @Override 67 | public T deserialize(byte[] buffer) { 68 | T v; 69 | try { 70 | ByteArrayInputStream stream = new ByteArrayInputStream(buffer); 71 | Hessian2Input is = new Hessian2Input(stream); 72 | is.setSerializerFactory(factory); 73 | v = (T) is.readObject(); 74 | is.close(); 75 | } catch (Exception e) { 76 | e.printStackTrace(); 77 | throw new RuntimeException(e); 78 | } 79 | return v; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/com/alok/diskmap/utils/ObjectConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.utils; 18 | 19 | import java.io.Serializable; 20 | 21 | public interface ObjectConverter { 22 | 23 | public byte[] serialize(Serializable object) throws Exception; 24 | 25 | public T deserialize(byte[] buffer); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/BasicOpsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap; 18 | 19 | import junit.framework.TestCase; 20 | 21 | import java.io.File; 22 | import java.io.Serializable; 23 | import java.util.Map; 24 | 25 | public class BasicOpsTest extends TestCase { 26 | private static final String TMP_DIR = "/tmp/tests"; 27 | // private static final String TMP_DIR = "/home/alok/sw_dev/tmp/tests"; 28 | 29 | public void setUp(){ 30 | File f = new File(TMP_DIR); 31 | if(!f.exists()){ 32 | f.mkdirs(); 33 | } 34 | } 35 | public void testSimplePut(){ 36 | String keyS = "test"; 37 | String value = "valueString"; 38 | String value2 = "valueString2"; 39 | Map map = getNBMap(); 40 | map.put(keyS, value); 41 | assertEquals(value, map.get(keyS)); 42 | map.put(keyS, value2); 43 | assertEquals(value2, map.get(keyS)); 44 | } 45 | 46 | public void testDelete() throws Exception{ 47 | int count = 10000; 48 | DiskBackedMap map = getNBMap(); 49 | for(int i = 0; i < count; i++){ 50 | map.put("Key" + i, "Value" + i); 51 | } 52 | long originalSize = map.sizeOnDisk(); 53 | for(int i = 0; i < count; i++){ 54 | if( i % 5 == 0){ 55 | map.remove("Key" + i); 56 | } 57 | } 58 | assertEquals(originalSize, map.sizeOnDisk()); 59 | map.gc(); 60 | assertTrue(originalSize > map.sizeOnDisk()); 61 | DiskBackedMap map2 = getMap(); 62 | for(int i = 0; i < count; i++){ 63 | if( i % 5 != 0){ 64 | assertEquals("Value" + i, map2.get("Key" + i)); 65 | } 66 | } 67 | } 68 | 69 | public void testHashCollisions(){ 70 | StringWithDuplicateHash str1 = new StringWithDuplicateHash("Foo", 1); 71 | StringWithDuplicateHash str2 = new StringWithDuplicateHash("Bar", 1); 72 | StringWithDuplicateHash str3 = new StringWithDuplicateHash("FooBar", 1); 73 | Map map = getNBMap(); 74 | map.put(str1, str1.getValue()); 75 | map.put(str2, str2.getValue()); 76 | assertEquals(str1.getValue(), map.get(str1)); 77 | assertEquals(str2.getValue(), map.get(str2)); 78 | assertNull(map.get(str3)); 79 | assertFalse(map.containsKey(str3)); 80 | } 81 | 82 | public static class StringWithDuplicateHash implements Serializable{ 83 | private String value; 84 | private int hash; 85 | 86 | public StringWithDuplicateHash(){ 87 | } 88 | public StringWithDuplicateHash(String value, int hash){ 89 | this.value = value; 90 | this.hash = hash; 91 | } 92 | 93 | public String getValue() { 94 | return value; 95 | } 96 | 97 | public void setValue(String value) { 98 | this.value = value; 99 | } 100 | 101 | public int getHash() { 102 | return hash; 103 | } 104 | 105 | public void setHash(int hash) { 106 | this.hash = hash; 107 | } 108 | 109 | @Override 110 | public int hashCode(){ 111 | return hash; 112 | } 113 | 114 | @Override 115 | public boolean equals(Object o) { 116 | if (this == o) return true; 117 | if (o == null || getClass() != o.getClass()) return false; 118 | 119 | StringWithDuplicateHash that = (StringWithDuplicateHash) o; 120 | 121 | if (value != null ? !value.equals(that.value) : that.value != null) return false; 122 | 123 | return true; 124 | } 125 | } 126 | 127 | private DiskBackedMap getMap() { 128 | return new DiskBackedMap(new Configuration().setDataDir(new File(TMP_DIR))); 129 | } 130 | 131 | private DiskBackedMap getNBMap() { 132 | return new DiskBackedMap(new Configuration().setDataDir(new File(TMP_DIR)).setUseNonBlockingReader(true)); 133 | } 134 | 135 | } 136 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/ConcurrentOpsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap; 18 | 19 | import junit.framework.Assert; 20 | import junit.framework.TestCase; 21 | 22 | import java.io.File; 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | import java.util.Map; 26 | import java.util.UUID; 27 | import java.util.concurrent.Callable; 28 | import java.util.concurrent.ExecutorService; 29 | import java.util.concurrent.Executors; 30 | 31 | public class ConcurrentOpsTest extends TestCase { 32 | private static final int THREAD_COUNT = 50; 33 | private static final String TEST_DIR = "/tmp/conc_tests"; 34 | 35 | public void setUp(){ 36 | File f = new File(TEST_DIR); 37 | if(!f.exists()){ 38 | f.mkdirs(); 39 | } 40 | } 41 | 42 | public void testConcurrentReadWrite() throws Exception{ 43 | DiskBackedMap map = new DiskBackedMap(TEST_DIR); 44 | ExecutorService executorService = Executors.newFixedThreadPool(THREAD_COUNT); 45 | 46 | List> tasks = new ArrayList>(); 47 | for(int i = 0; i < THREAD_COUNT * 10; i++){ 48 | tasks.add(new ReaderWriter(map, 10000)); 49 | if(i % THREAD_COUNT == 0){ 50 | tasks.add(new Vacummer(map)); 51 | } 52 | } 53 | executorService.invokeAll(tasks); 54 | } 55 | 56 | public class ReaderWriter implements Callable{ 57 | private Map map; 58 | private long count; 59 | 60 | public ReaderWriter(Map map, long count){ 61 | this.map = map; 62 | this.count = count; 63 | } 64 | 65 | @Override 66 | public Boolean call() throws Exception { 67 | try{ 68 | for(int i = 0; i < count; i++){ 69 | String key = UUID.randomUUID().toString(); 70 | String value = "Abcd" + key; 71 | String actualValue = null; 72 | try{ 73 | map.put(key, value); 74 | actualValue = map.get(key); 75 | }catch(Exception e) { 76 | // System.out.println("Put failed"); 77 | } 78 | Assert.assertEquals(actualValue, value); 79 | // if(i % 100 == 0){ 80 | // System.out.println("Removing key:" + key); 81 | // map.remove(key); 82 | // } 83 | } 84 | }catch(Exception e){ 85 | e.printStackTrace(); 86 | } 87 | return true; 88 | } 89 | } 90 | 91 | public class Vacummer implements Callable{ 92 | private DiskBackedMap map; 93 | 94 | public Vacummer (DiskBackedMap map){ 95 | this.map = map; 96 | } 97 | 98 | @Override 99 | public Boolean call() throws Exception { 100 | map.gc(); 101 | return true; 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/ConversionUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap; 18 | 19 | import junit.framework.TestCase; 20 | 21 | public class ConversionUtilsTest extends TestCase{ 22 | private ConversionUtils util; 23 | public void setUp(){ 24 | this.util = new ConversionUtils(); 25 | } 26 | 27 | public void testIntConversion(){ 28 | for(int i = 0; i < 1000000; i++){ 29 | int k = (int) (Math.random() * Integer.MAX_VALUE); 30 | int j = (int) (Math.random() * Integer.MIN_VALUE); 31 | assertEquals(k, util.byteToInt(util.intToBytes(k))); 32 | assertEquals(j, util.byteToInt(util.intToBytes(j))); 33 | assertEquals(i, util.byteToInt(util.intToBytes(i))); 34 | } 35 | } 36 | 37 | public void testShortConversion(){ 38 | for(int i = 0; i < Short.MAX_VALUE; i++){ 39 | int k = (int) (Math.random() * Short.MAX_VALUE); 40 | int j = (int) (Math.random() * Short.MIN_VALUE); 41 | assertEquals(k, util.byteToShort(util.shortToBytes((short) k))); 42 | assertEquals(j, util.byteToShort(util.shortToBytes((short) j))); 43 | assertEquals(i, util.byteToShort(util.shortToBytes((short) i))); 44 | } 45 | } 46 | 47 | public void testCrc(){ 48 | short crc = util.crc16("foo".getBytes()); 49 | byte[] bytes = util.shortToBytes(crc); 50 | assertEquals(crc, util.byteToShort(bytes)); 51 | } 52 | 53 | public void testLongConversion(){ 54 | for(long i = 0; i < 100000; i++){ 55 | long k = (long) (Math.random() * Long.MAX_VALUE); 56 | long j = (long) (Math.random() * Long.MIN_VALUE); 57 | assertEquals(k, util.byteToLong(util.longToBytes(k))); 58 | assertEquals(j, util.byteToLong(util.longToBytes(j))); 59 | assertEquals(i, util.byteToLong(util.longToBytes(i))); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/PageTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap; 18 | 19 | import junit.framework.TestCase; 20 | 21 | import java.io.File; 22 | 23 | public class PageTest extends TestCase { 24 | public void testLookup(){ 25 | Page page = new Page(new File("/home/alok/sw_dev/tmp"), 1); 26 | int count = 5000; 27 | for(int i = 0; i < count; i++){ 28 | page.save("key" + i, "value" + i ); 29 | } 30 | for(int i = 0; i < count; i++){ 31 | int key = (int)(Math.random() * count); 32 | String value = page.load("key" + key); 33 | System.out.println(String.format("Key[%s], Value[%s]", "key" + key, value)); 34 | assertEquals("value" + key, value); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/RBTreeTest.java: -------------------------------------------------------------------------------- 1 | package com.alok.diskmap; 2 | 3 | import junit.framework.TestCase; 4 | 5 | import java.io.*; 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | public class RBTreeTest extends TestCase{ 10 | 11 | public void testSerialization() throws IOException, ClassNotFoundException { 12 | RBTree rbTree = new RBTree(); 13 | List keys = new ArrayList(); 14 | for(int i = 0; i < 1000; i++){ 15 | int key = (int) (Math.random() * 10000); 16 | rbTree.insert(key, (long) (Math.random() * 10000)); 17 | keys.add(key); 18 | } 19 | ByteArrayOutputStream buffer = new ByteArrayOutputStream(); 20 | ObjectOutputStream o = new ObjectOutputStream(buffer); 21 | o.writeObject(rbTree); 22 | ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(buffer.toByteArray())); 23 | RBTree rbTree2 = (RBTree) in.readObject(); 24 | for (Integer key : keys) { 25 | long[] values = rbTree.lookup(key); 26 | long[] values2 = rbTree2.lookup(key); 27 | for (int i = 0; i < values.length; i++) { 28 | assertEquals(values[i], values2[i]); 29 | } 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/RecordTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap; 18 | 19 | import junit.framework.TestCase; 20 | 21 | import java.io.*; 22 | 23 | public class RecordTest extends TestCase{ 24 | public void testReadWrite() throws IOException{ 25 | String key = "foo"; 26 | String value = "bar"; 27 | Record r1 = new Record(key.getBytes(), value.getBytes(), Record.ACTIVE, key.hashCode(), 0); 28 | ByteArrayOutputStream buffer = new ByteArrayOutputStream(); 29 | DataOutput out = new DataOutputStream(buffer); 30 | r1.write(out); 31 | buffer.close(); 32 | Record r2 = new Record(); 33 | DataInput in = new DataInputStream(new ByteArrayInputStream(buffer.toByteArray())); 34 | r2.read(in); 35 | assertEquals(r1, r2); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/ZipMapTest.java: -------------------------------------------------------------------------------- 1 | package com.alok.diskmap; 2 | 3 | import junit.framework.TestCase; 4 | 5 | import java.util.*; 6 | 7 | public class ZipMapTest extends TestCase{ 8 | public void testZipMap(){ 9 | Map data = new HashMap(); 10 | ZipMap zipMap = new ZipMap(); 11 | for(int i = 0; i < 1000; i++){ 12 | String key = UUID.randomUUID().toString(); 13 | String value = UUID.randomUUID().toString(); 14 | data.put(key, value); 15 | zipMap.put(key, value.getBytes()); 16 | } 17 | for (String key : data.keySet()) { 18 | assertEquals(data.get(key), new String(zipMap.get(key))); 19 | } 20 | for (String key : zipMap.keySet()) { 21 | assertEquals(data.get(key), new String(zipMap.get(key))); 22 | } 23 | for (Map.Entry entry: zipMap.entrySet()) { 24 | assertEquals(data.get(entry.getKey()), new String(zipMap.get(entry.getKey()))); 25 | } 26 | for (String key : data.keySet()) { 27 | zipMap.remove(key); 28 | assertNull(zipMap.get(key)); 29 | } 30 | 31 | } 32 | 33 | public void testMapMemUsage(){ 34 | Map data = new HashMap(); 35 | int i = 0; 36 | while(true){ 37 | String key = UUID.randomUUID().toString(); 38 | String value = UUID.randomUUID().toString(); 39 | data.put(key, value.getBytes()); 40 | i++; 41 | if(i%10000 == 0){ 42 | System.out.println("Size:" + i); 43 | } 44 | } 45 | } 46 | 47 | public void testZipMapMemReadUsage(){ 48 | List maps = new ArrayList(); 49 | int mapCount = 50000; 50 | for(int i = 0; i < mapCount; i++){ 51 | maps.add(new ZipMap()); 52 | } 53 | int i = 0; 54 | int size = 0; 55 | long start = System.currentTimeMillis(); 56 | String value = UUID.randomUUID().toString() + UUID.randomUUID().toString(); 57 | String key = UUID.randomUUID().toString(); 58 | while(i < 3*1000*1000){ 59 | maps.get(i% mapCount).put(key + i, value.getBytes()); 60 | i++; 61 | size += key.length() + value.length(); 62 | if(i%10000 == 0){ 63 | long time = System.currentTimeMillis() - start; 64 | System.out.println("Count:" + i + ", size:" + size + ", avg time:" + (time)); 65 | start = System.currentTimeMillis(); 66 | } 67 | } 68 | i = 0; 69 | start = System.currentTimeMillis(); 70 | while(i < 3*1000*1000){ 71 | byte[] bytes = maps.get(i % mapCount).get(key + i); 72 | if(bytes == null){ 73 | System.out.println("Error"); 74 | } 75 | i++; 76 | if(i%10000 == 0){ 77 | long time = System.currentTimeMillis() - start; 78 | System.out.println("Count:" + i + ", size:" + size + ", avg read time:" + (time)); 79 | start = System.currentTimeMillis(); 80 | } 81 | } 82 | 83 | } 84 | 85 | public void testZipMapMemUsage(){ 86 | List maps = new ArrayList(); 87 | int mapCount = 50000; 88 | for(int i = 0; i < mapCount; i++){ 89 | maps.add(new ZipMap()); 90 | } 91 | int i = 0; 92 | int size = 0; 93 | long start = System.currentTimeMillis(); 94 | String value = UUID.randomUUID().toString() + UUID.randomUUID().toString(); 95 | String key = UUID.randomUUID().toString(); 96 | while(true){ 97 | maps.get(i% mapCount).put(key + i, value.getBytes()); 98 | i++; 99 | size += key.length() + value.length(); 100 | if(i%10000 == 0){ 101 | long time = System.currentTimeMillis() - start; 102 | System.out.println("Count:" + i + ", size:" + size + ", avg time:" + (time)); 103 | start = System.currentTimeMillis(); 104 | } 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/mem/MemoryUsageTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.mem; 18 | 19 | import com.alok.diskmap.Configuration; 20 | import com.alok.diskmap.DiskBackedMap; 21 | import com.sun.org.apache.xpath.internal.operations.Bool; 22 | import junit.framework.TestCase; 23 | import org.junit.Test; 24 | 25 | import java.io.File; 26 | import java.util.*; 27 | import java.util.concurrent.*; 28 | 29 | public class MemoryUsageTest extends TestCase { 30 | public void testLargeArray(){ 31 | String[] array = new String[100*1000*1000]; 32 | for(int i = 0; i < array.length; i++){ 33 | array[i] = UUID.randomUUID().toString(); 34 | } 35 | } 36 | 37 | public void testHashMapMemoryUsage(){ 38 | long startMemory = Runtime.getRuntime().totalMemory(); 39 | Map map = new HashMap(); 40 | for(int i = 1; i < Integer.MAX_VALUE; i++){ 41 | if(i %10000 == 0){ 42 | System.out.println("Average Used Memory:" + (Runtime.getRuntime().totalMemory())/i); 43 | System.out.println("entries:" + i); 44 | } 45 | map.put(new Integer(i), "Abcdefghijklmnopqrstuvwxyz" + Math.random()); 46 | } 47 | } 48 | 49 | public void testDiskMapMemoryUsage(){ 50 | long startMemory = Runtime.getRuntime().totalMemory(); 51 | Configuration configuration = new Configuration(); 52 | configuration.setDataDir(new File("/tmp/tests")).setFlushInterval(20000); 53 | Map map = new DiskBackedMap(configuration); 54 | long start = System.currentTimeMillis(); 55 | long loopStart = System.currentTimeMillis(); 56 | final int items = 100 * 1000 * 1000; 57 | final int loopItems = 200000; 58 | for(int i = 1; i < items; i++){ 59 | if(i % loopItems == 0){ 60 | System.out.println("Average Used Memory:" + (Runtime.getRuntime().totalMemory())/i); 61 | System.out.println(String.format("entries: %(,d", i)); 62 | System.out.println(String.format("Loop time: %(,d", (System.currentTimeMillis() - loopStart))); 63 | System.out.println(String.format("Total time: %(,d", System.currentTimeMillis() - start)); 64 | loopStart = System.currentTimeMillis(); 65 | } 66 | String key = "App-user-" + i + "-tag-" + (i % 5); 67 | map.put(key, "Abcdefghijklmnopqrstuvwxyz" + Math.random()); 68 | } 69 | long code = 0; 70 | 71 | } 72 | 73 | public void testConcurrentLookup(){ 74 | final long start = System.currentTimeMillis(); 75 | final int items = 100 * 1000 * 1000; 76 | final int loopItems = 200000; 77 | Configuration configuration = new Configuration(); 78 | configuration.setDataDir(new File("/tmp/tests")).setFlushInterval(20000); 79 | final Map map = new DiskBackedMap(configuration); 80 | Callable reader = new Callable() { 81 | @Override 82 | public Boolean call() { 83 | long loopStart = System.currentTimeMillis(); 84 | long code = 0; 85 | for(int i = 1; i < items; i++){ 86 | if(i % loopItems == 0){ 87 | System.out.println("Average Lookuptime:" + (System.currentTimeMillis() - start)/i); 88 | System.out.println(String.format("entries: %(,d", i)); 89 | System.out.println(String.format("Loop time: %(,d", (System.currentTimeMillis() - loopStart))); 90 | System.out.println(String.format("Total time: %(,d", System.currentTimeMillis() - start)); 91 | loopStart = System.currentTimeMillis(); 92 | } 93 | String key = "App-user-" + i + "-tag-" + (i % 5); 94 | String value = map.get(key); 95 | if(value != null){ 96 | code += value.hashCode(); 97 | }else{ 98 | System.out.println("Missed :" + key); 99 | } 100 | } 101 | return true; 102 | } 103 | }; 104 | List> futures = new ArrayList>(); 105 | ExecutorService executorService = Executors.newFixedThreadPool(10); 106 | for(int i = 0; i < 10; i++){ 107 | futures.add(executorService.submit(reader)); 108 | } 109 | for (Future future : futures) { 110 | try { 111 | future.get(); 112 | } catch (InterruptedException e) { 113 | e.printStackTrace(); 114 | } catch (ExecutionException e) { 115 | e.printStackTrace(); 116 | } 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/mock/MockObjectWithBinaryData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.mock; 18 | 19 | /** 20 | * Created by IntelliJ IDEA. 21 | * User: aloksingh 22 | * Date: Feb 27, 2010 23 | * Time: 4:30:14 PM 24 | * To change this template use File | Settings | File Templates. 25 | */ 26 | public class MockObjectWithBinaryData { 27 | } 28 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/mock/MockSimpleObject.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.mock; 18 | 19 | import java.io.Serializable; 20 | import java.math.BigDecimal; 21 | import java.math.BigInteger; 22 | import java.util.ArrayList; 23 | import java.util.Arrays; 24 | import java.util.List; 25 | import java.util.UUID; 26 | 27 | public class MockSimpleObject implements Serializable { 28 | private String a; 29 | private long b; 30 | private int c; 31 | private float d; 32 | private double e; 33 | private String[] f; 34 | private List g; 35 | private short h; 36 | private BigInteger i; 37 | private BigDecimal j; 38 | public MockSimpleObject(){ 39 | a = UUID.randomUUID().toString(); 40 | b = (long) (1000000 * Math.random()); 41 | c = (int) (1000000 * Math.random()); 42 | d = (float) (1000000f * Math.random()); 43 | e = (1000000 * Math.random()); 44 | f = new String[(int) (1000 * Math.random() + 1)]; 45 | for (int i = 0; i < f.length; i++) { 46 | f[i] = UUID.randomUUID().toString(); 47 | } 48 | g = new ArrayList(); 49 | for (int i = 0; i < f.length; i++) { 50 | g.add((long) (1000000 * Math.random())); 51 | } 52 | h = (short) (10000 * Math.random()); 53 | i = new BigInteger(String.valueOf((long)(1000000 * Math.random()))); 54 | j = new BigDecimal(String.valueOf(100000000 * Math.random())); 55 | } 56 | 57 | public String getA() { 58 | return a; 59 | } 60 | 61 | public void setA(String a) { 62 | this.a = a; 63 | } 64 | 65 | public long getB() { 66 | return b; 67 | } 68 | 69 | public void setB(long b) { 70 | this.b = b; 71 | } 72 | 73 | public int getC() { 74 | return c; 75 | } 76 | 77 | public void setC(int c) { 78 | this.c = c; 79 | } 80 | 81 | public float getD() { 82 | return d; 83 | } 84 | 85 | public void setD(float d) { 86 | this.d = d; 87 | } 88 | 89 | public double getE() { 90 | return e; 91 | } 92 | 93 | public void setE(double e) { 94 | this.e = e; 95 | } 96 | 97 | public String[] getF() { 98 | return f; 99 | } 100 | 101 | public void setF(String[] f) { 102 | this.f = f; 103 | } 104 | 105 | public List getG() { 106 | return g; 107 | } 108 | 109 | public void setG(List g) { 110 | this.g = g; 111 | } 112 | 113 | public short getH() { 114 | return h; 115 | } 116 | 117 | public void setH(short h) { 118 | this.h = h; 119 | } 120 | 121 | public BigInteger getI() { 122 | return i; 123 | } 124 | 125 | public void setI(BigInteger i) { 126 | this.i = i; 127 | } 128 | 129 | public BigDecimal getJ() { 130 | return j; 131 | } 132 | 133 | public void setJ(BigDecimal j) { 134 | this.j = j; 135 | } 136 | 137 | @Override 138 | public boolean equals(Object o) { 139 | if (this == o) return true; 140 | if (o == null || getClass() != o.getClass()) return false; 141 | 142 | MockSimpleObject that = (MockSimpleObject) o; 143 | 144 | if (b != that.b) { 145 | return false; 146 | } 147 | if (c != that.c) { 148 | return false; 149 | } 150 | if (Float.compare(that.d, d) != 0) { 151 | return false; 152 | } 153 | if (Double.compare(that.e, e) != 0) { 154 | return false; 155 | } 156 | if (h != that.h) { 157 | return false; 158 | } 159 | if (a != null ? !a.equals(that.a) : that.a != null) { 160 | return false; 161 | } 162 | if (!Arrays.equals(f, that.f)) { 163 | return false; 164 | } 165 | if (g != null ? !g.equals(that.g) : that.g != null) { 166 | return false; 167 | } 168 | if (i != null ? !i.equals(that.i) : that.i != null) { 169 | return false; 170 | } 171 | if (j != null ? (j.compareTo(that.j) != 0 ): that.j != null) { 172 | return false; 173 | } 174 | 175 | return true; 176 | } 177 | 178 | @Override 179 | public int hashCode() { 180 | int result; 181 | long temp; 182 | result = a != null ? a.hashCode() : 0; 183 | result = 31 * result + (int) (b ^ (b >>> 32)); 184 | result = 31 * result + c; 185 | result = 31 * result + (d != +0.0f ? Float.floatToIntBits(d) : 0); 186 | temp = e != +0.0d ? Double.doubleToLongBits(e) : 0L; 187 | result = 31 * result + (int) (temp ^ (temp >>> 32)); 188 | result = 31 * result + (f != null ? Arrays.hashCode(f) : 0); 189 | result = 31 * result + (g != null ? g.hashCode() : 0); 190 | result = 31 * result + (int) h; 191 | result = 31 * result + (i != null ? i.hashCode() : 0); 192 | result = 31 * result + (j != null ? j.hashCode() : 0); 193 | return result; 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/perf/Config.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.perf; 18 | 19 | import java.io.File; 20 | 21 | public class Config { 22 | private int itemCount; 23 | private int runCount; 24 | private int step; 25 | private String statsDir; 26 | private String name; 27 | 28 | public Config(String name, Integer itemCount, Integer runCount, Integer step, String statsDir) { 29 | this.name = name; 30 | this.itemCount = itemCount; 31 | this.runCount = runCount; 32 | this.step = step; 33 | this.statsDir = statsDir; 34 | } 35 | 36 | public int getItemCount() { 37 | return itemCount; 38 | } 39 | 40 | public int getRunCount() { 41 | return runCount; 42 | } 43 | 44 | public int getStep() { 45 | return step; 46 | } 47 | 48 | public String getStatsDir() { 49 | return statsDir + File.separator + getName().replace(' ', '_') + ".txt"; 50 | } 51 | 52 | public String getName() { 53 | return name; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/perf/Driver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.perf; 18 | 19 | import com.alok.diskmap.Configuration; 20 | import com.alok.diskmap.DiskBackedMap; 21 | 22 | import java.io.File; 23 | import java.io.Serializable; 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | import java.util.Map; 27 | 28 | public class Driver { 29 | public static void main(String[] args) throws Exception { 30 | Config cfg = new Config(args[0], Integer.parseInt(args[1]), Integer.parseInt(args[2]), Integer.parseInt(args[3]), args[4]); 31 | if(cfg.getName().startsWith("DiskBackedMap-Reader-Threads-1")){ 32 | Map map = new DiskBackedMap(new Configuration().setDataDir(new File(args[5])).setUseNonBlockingReader(true)); 33 | IncrementingKeyValueGen generator = new IncrementingKeyValueGen(); 34 | // populateData(map, cfg.getItemCount(), generator); 35 | StatsCollector collector = new StatsCollector(cfg); 36 | collector.writeHeader(); 37 | PerfTask task = new Reader(map, cfg, collector, generator); 38 | task.run(); 39 | collector.close(); 40 | }else { 41 | 42 | Map map = new DiskBackedMap(new Configuration().setDataDir(new File(args[5])).setUseNonBlockingReader(true)); 43 | IncrementingKeyValueGen generator = new IncrementingKeyValueGen(); 44 | populateData(map, cfg.getItemCount(), generator); 45 | List threads = new ArrayList(); 46 | StatsCollector collector = new StatsCollector(cfg); 47 | collector.writeHeader(); 48 | int THREAD_COUNT = 2; 49 | if(cfg.getName().startsWith("DiskBackedMap-Reader-Threads-2")){ 50 | THREAD_COUNT = 2; 51 | } 52 | if(cfg.getName().startsWith("DiskBackedMap-Reader-Threads-4")){ 53 | THREAD_COUNT = 4; 54 | } 55 | if(cfg.getName().startsWith("DiskBackedMap-Reader-Threads-6")){ 56 | THREAD_COUNT = 6; 57 | } 58 | for(int i = 0; i < THREAD_COUNT; i++){ 59 | Thread t = new Thread(new Reader(map, cfg, collector, generator)); 60 | threads.add(t); 61 | t.start(); 62 | } 63 | for (Thread thread : threads) { 64 | try { 65 | thread.join(); 66 | } catch (InterruptedException e) { 67 | e.printStackTrace(); 68 | } 69 | } 70 | collector.close(); 71 | ((DiskBackedMap) map).close(); 72 | } 73 | } 74 | 75 | private static void populateData(Map map, int itemCount, KeyValueGen generator) { 76 | long time = System.currentTimeMillis(); 77 | for(int i = 0; i < itemCount; i++){ 78 | map.put(generator.nextKey(), generator.nextValue()); 79 | if(i % (itemCount/100) == 0){ 80 | time = System.currentTimeMillis() - time; 81 | System.out.println(String.format("Generated [%d] of [%d] in [%d] ms", i, itemCount, time)); 82 | time = System.currentTimeMillis(); 83 | } 84 | } 85 | } 86 | public static class IncrementingKeyValueGen implements KeyValueGen{ 87 | private int currentKey = 0; 88 | 89 | public IncrementingKeyValueGen(int cKey){ 90 | this.currentKey = cKey; 91 | } 92 | 93 | public IncrementingKeyValueGen(){ 94 | this.currentKey = 0; 95 | } 96 | 97 | @Override 98 | public Serializable nextKey() { 99 | return new Integer(currentKey++); 100 | } 101 | 102 | @Override 103 | public Serializable nextValue() { 104 | return String.format("Value[ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-%d",currentKey); 105 | } 106 | 107 | @Override 108 | public Serializable existingKey() { 109 | return new Integer((int) (Math.random() * currentKey)); 110 | } 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/perf/KeyValueGen.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.perf; 18 | 19 | import java.io.Serializable; 20 | 21 | public interface KeyValueGen { 22 | Serializable nextKey(); 23 | 24 | Serializable nextValue(); 25 | 26 | Serializable existingKey(); 27 | } 28 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/perf/PerfTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.perf; 18 | 19 | import java.util.Map; 20 | 21 | public abstract class PerfTask implements Runnable{ 22 | private Map map; 23 | private Config config; 24 | private StatsCollector statsCollector; 25 | private KeyValueGen generator; 26 | 27 | public PerfTask(Map map, StatsCollector statsCollector, Config config, KeyValueGen generator) { 28 | this.map = map; 29 | this.statsCollector = statsCollector; 30 | this.config = config; 31 | this.generator = generator; 32 | } 33 | 34 | public void run(){ 35 | for(int i = 0; i < config.getRunCount(); i = i + config.getStep()){ 36 | execute(statsCollector, config.getStep()); 37 | } 38 | } 39 | 40 | public Map getMap() { 41 | return map; 42 | } 43 | 44 | public Config getConfig() { 45 | return config; 46 | } 47 | 48 | public StatsCollector getStatsCollector() { 49 | return statsCollector; 50 | } 51 | 52 | public KeyValueGen getGenerator() { 53 | return generator; 54 | } 55 | 56 | protected abstract Object execute(StatsCollector statsCollector, int step); 57 | } 58 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/perf/Reader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.alok.diskmap.perf; 17 | 18 | import java.io.Serializable; 19 | import java.util.Map; 20 | 21 | public class Reader extends PerfTask{ 22 | 23 | public Reader(Map map, Config config, StatsCollector statsCollector, KeyValueGen generator){ 24 | super(map, statsCollector, config, generator); 25 | } 26 | 27 | @Override 28 | protected Object execute(StatsCollector statsCollector, int step) { 29 | long time = 0; 30 | long hash = 0; 31 | long missCount = 0; 32 | for(int i = 0; i < step; i++){ 33 | long stepTime = System.nanoTime(); 34 | Serializable key = getGenerator().existingKey(); 35 | Object value = getMap().get(key); 36 | stepTime = System.nanoTime() - stepTime; 37 | time+= stepTime; 38 | hash += (value != null ? value.hashCode() : 0); 39 | if(value == null || value.toString().length() < 36){ 40 | missCount++; 41 | } 42 | } 43 | statsCollector.update(step, time); 44 | System.out.println(String.format("Missed %d out of %d", missCount, step)); 45 | return hash; 46 | } 47 | 48 | 49 | } 50 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/perf/StatsCollector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.perf; 18 | 19 | import java.io.FileWriter; 20 | import java.io.IOException; 21 | import java.io.Writer; 22 | 23 | public class StatsCollector { 24 | private Config config; 25 | private Writer writer; 26 | 27 | public StatsCollector(Config config) { 28 | this.config = config; 29 | try { 30 | this.writer = new FileWriter(this.config.getStatsDir()); 31 | } catch (IOException e) { 32 | throw new RuntimeException(e); 33 | } 34 | } 35 | public void writeHeader(){ 36 | synchronized (writer){ 37 | try { 38 | writer.write(String.format("Name [%s], Total Items[%d], Total Operations[%d]", config.getName(), config.getItemCount(), config.getRunCount())); 39 | writer.write("\n"); 40 | writer.write("Items,Time"); 41 | writer.write("\n"); 42 | } catch (IOException e) { 43 | throw new RuntimeException(e); 44 | } 45 | } 46 | } 47 | 48 | public void close(){ 49 | synchronized (writer){ 50 | try { 51 | writer.close(); 52 | } catch (IOException e) { 53 | e.printStackTrace(); 54 | } 55 | } 56 | } 57 | public void update(int step, long time) { 58 | synchronized (writer){ 59 | try { 60 | writer.write(String.format("%d,%d\n", step, time)); 61 | } catch (IOException e) { 62 | throw new RuntimeException(e); 63 | } 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /test/com/alok/diskmap/utils/ObjectConversionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009 Alok Singh 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alok.diskmap.utils; 18 | 19 | import com.alok.diskmap.mock.MockSimpleObject; 20 | import junit.framework.TestCase; 21 | 22 | import java.math.BigDecimal; 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | 26 | public class ObjectConversionTest extends TestCase { 27 | public void testConversionSpeed() throws Exception { 28 | List objects = new ArrayList(); 29 | for(int i = 0; i < 1000; i++){ 30 | objects.add(new MockSimpleObject()); 31 | } 32 | roundTrip(objects, new Hessian2ObjectConverter()); 33 | roundTrip(objects, new DefaultObjectConverter()); 34 | } 35 | 36 | private void roundTrip(List objects, ObjectConverter o1) throws Exception { 37 | List buffers = serialize(objects, o1); 38 | long size = 0; 39 | for (byte[] buffer : buffers) { 40 | size+= buffer.length; 41 | } 42 | System.out.println("Size:" + size); 43 | List newObjects = deserialize(buffers, o1); 44 | for (int i = 0; i < objects.size(); i++) { 45 | assertTrue(objects.get(i).equals(newObjects.get(i))); 46 | } 47 | } 48 | 49 | private List serialize(List objects, ObjectConverter converter) throws Exception { 50 | List buffers = new ArrayList(objects.size()); 51 | long time = System.currentTimeMillis(); 52 | for (MockSimpleObject object : objects) { 53 | buffers.add(converter.serialize(object)); 54 | } 55 | time = System.currentTimeMillis() - time; 56 | System.out.println(converter.getClass().getName() + " serialization time:" + time); 57 | return buffers; 58 | } 59 | 60 | private List deserialize(List buffers, ObjectConverter converter) throws Exception { 61 | List objects = new ArrayList(buffers.size()); 62 | long time = System.currentTimeMillis(); 63 | for (byte[] buffer : buffers) { 64 | objects.add(converter.deserialize(buffer)); 65 | } 66 | time = System.currentTimeMillis() - time; 67 | System.out.println(converter.getClass().getName() + " deserialization time:" + time); 68 | return objects; 69 | } 70 | 71 | public void testBigDecimal() throws Exception { 72 | ObjectConverter o1 = new Hessian2ObjectConverter(); 73 | BigDecimal bigDecimal = new BigDecimal("12474639.945458954"); 74 | byte[] bytes = o1.serialize(bigDecimal); 75 | BigDecimal newDecimal = o1.deserialize(bytes); 76 | assertEquals(bigDecimal, newDecimal); 77 | } 78 | } 79 | --------------------------------------------------------------------------------