├── .gitignore ├── .travis.yml ├── README.md ├── pom.xml └── src ├── main └── java │ └── com │ └── blogspot │ └── mydailyjava │ └── guava │ └── cache │ └── overflow │ ├── AbstractLoadingPersistingCache.java │ ├── AbstractPersistingCache.java │ ├── FileSystemCacheBuilder.java │ ├── FileSystemLoadingPersistingCache.java │ ├── FileSystemPersistingCache.java │ ├── NotPersistedException.java │ └── RemovalNotifications.java └── test └── java └── com └── blogspot └── mydailyjava └── guava └── cache └── overflow ├── FileSystemLoadingPersistingCacheTest.java ├── FileSystemPersistingCacheTest.java ├── KeyValuePair.java └── RemovalListenerTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | # Eclipse 2 | .classpath 3 | .project 4 | .settings/ 5 | 6 | # Intellij 7 | .idea/ 8 | *.iml 9 | *.iws 10 | 11 | # Mac 12 | .DS_Store 13 | 14 | # Maven 15 | log/ 16 | target/ 17 | site/ 18 | 19 | #jRebel 20 | rebel.xml -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | A Guava cache extension that allows caches to persist cache entries when they cannot longer be stored in memory. 2 | An implementation that overflows to the file system is provided including a corresponding `CacheBuilder` with similar semantics than the Guava `CacheBuilder`. 3 | 4 | For creating a cache that overflows to disk, just proceed as when using the Guava CacheBuilder: 5 | 6 | ```java 7 | Cache stringCache = 8 | FileSystemCacheBuilder.newBuilder() 9 | .maximumSize(100L) 10 | .softValues() 11 | .build(); 12 | ``` 13 | 14 | **Note**: This cache implementation has slightly different semantics than the `Cache` / `LoadingCache` interface contracts specify: 15 | * Any limits set for this cache do only concern the cache's memory size. Cache entries exceeding this limit will overflow to disk. 16 | * When calling the non-argument `invalidateAll()` method, the RemovalListener is only informed about the expiration of entries that are still stored in memory. 17 | * When the cache is not longer in use, its `invalidateAll()` method should be called if the cache's overflow folder is not cleared by the operating system. 18 | * There is a minimal risk of concurrency issues since cache entries are still accessible when the `RemovalListener` which is responsible for serializing the cache entry writes the entry to disk. This problem does not matter for immutable cache objects, but mutable state might get lost when cache entries are retreived while they are serialized. 19 | 20 | Licensed under the Apache Software License, Version 2.0 21 | 22 | [![Build Status](https://travis-ci.org/raphw/guava-cache-overflow-extension.png)](https://travis-ci.org/raphw/guava-cache-overflow-extension) 23 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.blogspot.mydailyjava 7 | guava-cache-overflow-extension 8 | 0.3-SNAPSHOT 9 | 10 | 2013 11 | 12 | Guava cache overflow extension 13 | An extension to Guava caches that allows cache entries to overflow to disk. 14 | https://github.com/raphw/guava-cache-overflow-extension 15 | 16 | 17 | 14.0.1 18 | 1.7.5 19 | 6.8.5 20 | 1.7.5 21 | 22 | 23 | 24 | 25 | The Apache Software License, Version 2.0 26 | http://www.apache.org/licenses/LICENSE-2.0.txt 27 | repo 28 | A business-friendly OSS license 29 | 30 | 31 | 32 | 33 | 34 | raphw 35 | Rafael Winterhalter 36 | rafael.wth@web.de 37 | http://mydailyjava.blogspot.com 38 | 39 | developer 40 | 41 | +1 42 | 43 | 44 | 45 | 46 | 47 | com.google.guava 48 | guava 49 | ${version.guava} 50 | 51 | 52 | org.slf4j 53 | slf4j-api 54 | ${version.slf4j} 55 | 56 | 57 | org.testng 58 | testng 59 | ${version.testng} 60 | test 61 | 62 | 63 | org.slf4j 64 | slf4j-simple 65 | ${version.slf4j.simple} 66 | test 67 | 68 | 69 | 70 | 71 | 72 | org.sonatype.oss 73 | oss-parent 74 | 7 75 | 76 | 77 | 78 | github.com 79 | https://github.com/raphw/guava-cache-overflow-extension/issues 80 | 81 | 82 | 83 | scm:git:git@github.com:raphw/guava-cache-overflow-extension.git 84 | scm:git:git@github.com:raphw/guava-cache-overflow-extension.git 85 | git@github.com:raphw/guava-cache-overflow-extension.git 86 | 87 | 88 | 89 | 90 | 91 | org.apache.maven.plugins 92 | maven-compiler-plugin 93 | 2.5.1 94 | true 95 | 96 | 1.6 97 | 1.6 98 | utf-8 99 | 100 | 101 | 102 | 103 | 104 | -------------------------------------------------------------------------------- /src/main/java/com/blogspot/mydailyjava/guava/cache/overflow/AbstractLoadingPersistingCache.java: -------------------------------------------------------------------------------- 1 | package com.blogspot.mydailyjava.guava.cache.overflow; 2 | 3 | import com.google.common.cache.CacheBuilder; 4 | import com.google.common.cache.CacheLoader; 5 | import com.google.common.cache.LoadingCache; 6 | import com.google.common.cache.RemovalListener; 7 | import com.google.common.collect.ImmutableMap; 8 | 9 | import java.util.concurrent.Callable; 10 | import java.util.concurrent.ExecutionException; 11 | 12 | public abstract class AbstractLoadingPersistingCache extends AbstractPersistingCache implements LoadingCache { 13 | 14 | private final CacheLoader cacheLoader; 15 | 16 | protected AbstractLoadingPersistingCache(CacheBuilder cacheBuilder, CacheLoader cacheLoader) { 17 | this(cacheBuilder, cacheLoader, null); 18 | } 19 | 20 | protected AbstractLoadingPersistingCache(CacheBuilder cacheBuilder, CacheLoader cacheLoader, RemovalListener removalListener) { 21 | super(cacheBuilder, removalListener); 22 | this.cacheLoader = cacheLoader; 23 | } 24 | 25 | private class ValueLoaderFromCacheLoader implements Callable { 26 | 27 | private final K key; 28 | private final CacheLoader cacheLoader; 29 | 30 | private ValueLoaderFromCacheLoader(CacheLoader cacheLoader, K key) { 31 | this.key = key; 32 | this.cacheLoader = cacheLoader; 33 | } 34 | 35 | @Override 36 | public V call() throws Exception { 37 | return cacheLoader.load(key); 38 | } 39 | } 40 | 41 | @Override 42 | public V get(K key) throws ExecutionException { 43 | return get(key, new ValueLoaderFromCacheLoader(cacheLoader, key)); 44 | } 45 | 46 | @Override 47 | public V getUnchecked(K key) { 48 | try { 49 | return get(key, new ValueLoaderFromCacheLoader(cacheLoader, key)); 50 | } catch (ExecutionException e) { 51 | throw new RuntimeException(e.getCause()); 52 | } 53 | } 54 | 55 | @Override 56 | public ImmutableMap getAll(Iterable keys) throws ExecutionException { 57 | ImmutableMap.Builder all = ImmutableMap.builder(); 58 | for (K key : keys) { 59 | all.put(key, get(key)); 60 | } 61 | return all.build(); 62 | } 63 | 64 | @Override 65 | public V apply(K key) { 66 | try { 67 | return cacheLoader.load(key); 68 | } catch (Exception e) { 69 | throw new RuntimeException(String.format("Could not apply cache on key %s", key), e); 70 | } 71 | } 72 | 73 | @Override 74 | public void refresh(K key) { 75 | try { 76 | getUnderlyingCache().put(key, cacheLoader.load(key)); 77 | } catch (Exception e) { 78 | throw new RuntimeException(String.format("Could not refresh value for key %s", key), e); 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/com/blogspot/mydailyjava/guava/cache/overflow/AbstractPersistingCache.java: -------------------------------------------------------------------------------- 1 | package com.blogspot.mydailyjava.guava.cache.overflow; 2 | 3 | import com.google.common.cache.*; 4 | import com.google.common.collect.ImmutableMap; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.io.IOException; 9 | import java.io.InputStream; 10 | import java.io.OutputStream; 11 | import java.util.List; 12 | import java.util.Map; 13 | import java.util.concurrent.Callable; 14 | import java.util.concurrent.ConcurrentMap; 15 | import java.util.concurrent.ExecutionException; 16 | 17 | public abstract class AbstractPersistingCache implements Cache { 18 | 19 | private static final Logger LOGGER = LoggerFactory.getLogger(AbstractPersistingCache.class); 20 | 21 | private final LoadingCache underlyingCache; 22 | private final RemovalListener removalListener; 23 | 24 | protected AbstractPersistingCache(CacheBuilder cacheBuilder) { 25 | this(cacheBuilder, null); 26 | } 27 | 28 | protected AbstractPersistingCache(CacheBuilder cacheBuilder, RemovalListener removalListener) { 29 | this.underlyingCache = makeCache(cacheBuilder); 30 | this.removalListener = removalListener; 31 | } 32 | 33 | private LoadingCache makeCache(CacheBuilder cacheBuilder) { 34 | return cacheBuilder 35 | .removalListener(new PersistingRemovalListener()) 36 | .build(new PersistedStateCacheLoader()); 37 | } 38 | 39 | private class PersistingRemovalListener implements RemovalListener { 40 | @Override 41 | public void onRemoval(RemovalNotification notification) { 42 | if (isPersistenceRelevant(notification.getCause())) { 43 | try { 44 | persistValue(notification.getKey(), notification.getValue()); 45 | } catch (IOException e) { 46 | LOGGER.warn(String.format("Could not persist value %s to key %s", 47 | notification.getKey(), notification.getValue()), e); 48 | } 49 | } else if (removalListener != null) { 50 | removalListener.onRemoval(notification); 51 | } 52 | } 53 | } 54 | 55 | private class PersistedStateCacheLoader extends CacheLoader { 56 | @Override 57 | public V load(K key) throws Exception { 58 | V value = null; 59 | try { 60 | value = findPersisted(key); 61 | if (value != null) { 62 | deletePersistedIfExistent(key); 63 | underlyingCache.put(key, value); 64 | } 65 | } catch (Exception e) { 66 | LOGGER.warn(String.format("Could not load persisted value to key %s", key), e); 67 | } 68 | if (value != null) { 69 | return value; 70 | } else { 71 | throw new NotPersistedException(); 72 | } 73 | } 74 | } 75 | 76 | private class PersistedStateValueLoader extends PersistedStateCacheLoader implements Callable { 77 | 78 | private final K key; 79 | private final Callable valueLoader; 80 | 81 | private PersistedStateValueLoader(K key, Callable valueLoader) { 82 | this.key = key; 83 | this.valueLoader = valueLoader; 84 | } 85 | 86 | @Override 87 | public V call() throws Exception { 88 | V value; 89 | try { 90 | value = load(key); 91 | } catch (NotPersistedException e) { 92 | value = null; 93 | } 94 | if (value != null) return value; 95 | return valueLoader.call(); 96 | } 97 | } 98 | 99 | protected boolean isPersistenceRelevant(RemovalCause removalCause) { 100 | // Note: RemovalCause#wasEvicted is package private 101 | return removalCause != RemovalCause.EXPLICIT 102 | && removalCause != RemovalCause.REPLACED; 103 | } 104 | 105 | protected LoadingCache getUnderlyingCache() { 106 | return underlyingCache; 107 | } 108 | 109 | protected abstract V findPersisted(K key) throws IOException; 110 | 111 | protected abstract void persistValue(K key, V value) throws IOException; 112 | 113 | protected abstract List directoryFor(K key); 114 | 115 | protected abstract void persist(K key, V value, OutputStream outputStream) throws IOException; 116 | 117 | protected abstract V readPersisted(K key, InputStream inputStream) throws IOException; 118 | 119 | protected abstract boolean isPersist(K key); 120 | 121 | protected abstract void deletePersistedIfExistent(K key); 122 | 123 | protected abstract void deleteAllPersisted(); 124 | 125 | protected abstract int sizeOfPersisted(); 126 | 127 | @Override 128 | @SuppressWarnings("unchecked") 129 | public V getIfPresent(Object key) { 130 | try { 131 | K castKey = (K) key; 132 | return underlyingCache.get(castKey); 133 | } catch (ClassCastException e) { 134 | LOGGER.info(String.format("Could not cast key %s to desired type", key), e); 135 | } catch (ExecutionException e) { 136 | LOGGER.warn(String.format("Persisted value to key %s could not be retrieved", key), e); 137 | throw new RuntimeException("Error while loading persisted value", e); 138 | } catch (RuntimeException e) { 139 | if (!(e.getCause() instanceof NotPersistedException)) { 140 | throw e; 141 | } 142 | } 143 | return null; 144 | } 145 | 146 | @Override 147 | public V get(K key, Callable valueLoader) throws ExecutionException { 148 | return underlyingCache.get(key, new PersistedStateValueLoader(key, valueLoader)); 149 | } 150 | 151 | @Override 152 | @SuppressWarnings("unchecked") 153 | public ImmutableMap getAllPresent(Iterable keys) { 154 | ImmutableMap.Builder allPresent = ImmutableMap.builder(); 155 | for (Object key : keys) { 156 | V value = getIfPresent(key); 157 | if (value != null) allPresent.put((K) key, value); 158 | } 159 | return allPresent.build(); 160 | } 161 | 162 | @Override 163 | public void put(K key, V value) { 164 | underlyingCache.put(key, value); 165 | } 166 | 167 | @Override 168 | public void putAll(Map m) { 169 | underlyingCache.putAll(m); 170 | } 171 | 172 | @Override 173 | public void invalidate(Object key) { 174 | underlyingCache.invalidate(key); 175 | invalidatePersisted(key); 176 | } 177 | 178 | @Override 179 | public void invalidateAll(Iterable keys) { 180 | underlyingCache.invalidateAll(keys); 181 | for (Object key : keys) { 182 | invalidatePersisted(key); 183 | } 184 | } 185 | 186 | @SuppressWarnings("unchecked") 187 | private void invalidatePersisted(Object key) { 188 | try { 189 | K castKey = (K) key; 190 | if (removalListener == null) { 191 | deletePersistedIfExistent(castKey); 192 | } else { 193 | V value = findPersisted(castKey); 194 | if (value != null) { 195 | removalListener.onRemoval(RemovalNotifications.make(castKey, value)); 196 | deletePersistedIfExistent(castKey); 197 | } 198 | } 199 | } catch (ClassCastException e) { 200 | LOGGER.info(String.format("Could not cast key %s to desired type", key), e); 201 | } catch (IOException e) { 202 | e.printStackTrace(); 203 | } 204 | } 205 | 206 | @Override 207 | public void invalidateAll() { 208 | underlyingCache.invalidateAll(); 209 | deleteAllPersisted(); 210 | } 211 | 212 | @Override 213 | public long size() { 214 | return underlyingCache.size() + sizeOfPersisted(); 215 | } 216 | 217 | @Override 218 | public CacheStats stats() { 219 | return underlyingCache.stats(); 220 | } 221 | 222 | @Override 223 | public ConcurrentMap asMap() { 224 | return underlyingCache.asMap(); 225 | } 226 | 227 | @Override 228 | public void cleanUp() { 229 | underlyingCache.cleanUp(); 230 | } 231 | } 232 | -------------------------------------------------------------------------------- /src/main/java/com/blogspot/mydailyjava/guava/cache/overflow/FileSystemCacheBuilder.java: -------------------------------------------------------------------------------- 1 | package com.blogspot.mydailyjava.guava.cache.overflow; 2 | 3 | import com.google.common.base.Ticker; 4 | import com.google.common.cache.*; 5 | 6 | import java.io.File; 7 | import java.util.concurrent.TimeUnit; 8 | 9 | import static com.google.common.base.Preconditions.checkNotNull; 10 | import static com.google.common.base.Preconditions.checkState; 11 | 12 | /** 13 | * {@link com.google.common.cache.CacheBuilder} 14 | */ 15 | public final class FileSystemCacheBuilder { 16 | 17 | /** 18 | * {@link com.google.common.cache.CacheBuilder#from(com.google.common.cache.CacheBuilderSpec)} 19 | */ 20 | public static FileSystemCacheBuilder from(CacheBuilderSpec spec) { 21 | return new FileSystemCacheBuilder(CacheBuilder.from(spec)); 22 | } 23 | 24 | /** 25 | * {@link com.google.common.cache.CacheBuilder#from(java.lang.String)} 26 | */ 27 | public static FileSystemCacheBuilder from(String spec) { 28 | return new FileSystemCacheBuilder(CacheBuilder.from(spec)); 29 | } 30 | 31 | /** 32 | * {@link com.google.common.cache.CacheBuilder#newBuilder()} 33 | */ 34 | public static FileSystemCacheBuilder newBuilder() { 35 | return new FileSystemCacheBuilder(); 36 | } 37 | 38 | private final CacheBuilder underlyingCacheBuilder; 39 | 40 | private RemovalListener removalListener; 41 | private File persistenceDirectory; 42 | 43 | private FileSystemCacheBuilder() { 44 | this.underlyingCacheBuilder = CacheBuilder.newBuilder(); 45 | } 46 | 47 | private FileSystemCacheBuilder(CacheBuilder cacheBuilder) { 48 | this.underlyingCacheBuilder = cacheBuilder; 49 | } 50 | 51 | /** 52 | * {@link com.google.common.cache.CacheBuilder#concurrencyLevel(int)} 53 | */ 54 | public FileSystemCacheBuilder concurrencyLevel(int concurrencyLevel) { 55 | underlyingCacheBuilder.concurrencyLevel(concurrencyLevel); 56 | return this; 57 | } 58 | 59 | /** 60 | * {@link com.google.common.cache.CacheBuilder#expireAfterAccess(long, TimeUnit)} 61 | */ 62 | public FileSystemCacheBuilder expireAfterAccess(long duration, TimeUnit unit) { 63 | underlyingCacheBuilder.expireAfterWrite(duration, unit); 64 | return this; 65 | } 66 | 67 | /** 68 | * {@link com.google.common.cache.CacheBuilder#expireAfterWrite(long, java.util.concurrent.TimeUnit)} 69 | */ 70 | public FileSystemCacheBuilder expireAfterWrite(long duration, TimeUnit unit) { 71 | underlyingCacheBuilder.expireAfterWrite(duration, unit); 72 | return this; 73 | } 74 | 75 | /** 76 | * {@link com.google.common.cache.CacheBuilder#refreshAfterWrite(long, java.util.concurrent.TimeUnit)} 77 | */ 78 | public FileSystemCacheBuilder refreshAfterWrite(long duration, TimeUnit unit) { 79 | underlyingCacheBuilder.refreshAfterWrite(duration, unit); 80 | return this; 81 | } 82 | 83 | /** 84 | * {@link com.google.common.cache.CacheBuilder#initialCapacity(int)} 85 | */ 86 | public FileSystemCacheBuilder initialCapacity(int initialCapacity) { 87 | underlyingCacheBuilder.initialCapacity(initialCapacity); 88 | return this; 89 | } 90 | 91 | /** 92 | * {@link com.google.common.cache.CacheBuilder#maximumSize(long)} 93 | */ 94 | public FileSystemCacheBuilder maximumSize(long size) { 95 | underlyingCacheBuilder.maximumSize(size); 96 | return this; 97 | } 98 | 99 | /** 100 | * {@link com.google.common.cache.CacheBuilder#maximumWeight(long)} 101 | */ 102 | public FileSystemCacheBuilder maximumWeight(long weight) { 103 | underlyingCacheBuilder.maximumWeight(weight); 104 | return this; 105 | } 106 | 107 | /** 108 | * {@link com.google.common.cache.CacheBuilder#recordStats()} 109 | */ 110 | public FileSystemCacheBuilder recordStats() { 111 | underlyingCacheBuilder.recordStats(); 112 | return this; 113 | } 114 | 115 | /** 116 | * {@link com.google.common.cache.CacheBuilder#softValues()} 117 | */ 118 | public FileSystemCacheBuilder softValues() { 119 | underlyingCacheBuilder.softValues(); 120 | return this; 121 | } 122 | 123 | /** 124 | * {@link com.google.common.cache.CacheBuilder#weakKeys()} 125 | */ 126 | public FileSystemCacheBuilder weakKeys() { 127 | underlyingCacheBuilder.weakKeys(); 128 | return this; 129 | } 130 | 131 | /** 132 | * {@link com.google.common.cache.CacheBuilder#weakValues()} 133 | */ 134 | public FileSystemCacheBuilder weakValues() { 135 | underlyingCacheBuilder.weakValues(); 136 | return this; 137 | } 138 | 139 | /** 140 | * {@link CacheBuilder#ticker(com.google.common.base.Ticker)} 141 | */ 142 | public FileSystemCacheBuilder ticker(Ticker ticker) { 143 | underlyingCacheBuilder.ticker(ticker); 144 | return this; 145 | } 146 | 147 | /** 148 | * {@link CacheBuilder#weigher(com.google.common.cache.Weigher)} 149 | */ 150 | @SuppressWarnings("unchecked") 151 | public FileSystemCacheBuilder weigher(Weigher weigher) { 152 | underlyingCacheBuilder.weigher(weigher); 153 | return (FileSystemCacheBuilder) this; 154 | } 155 | 156 | /** 157 | * {@link CacheBuilder#removalListener(com.google.common.cache.RemovalListener)} 158 | */ 159 | public FileSystemCacheBuilder removalListener(RemovalListener listener) { 160 | checkState(this.removalListener == null); 161 | @SuppressWarnings("unchecked") 162 | FileSystemCacheBuilder castThis = (FileSystemCacheBuilder) this; 163 | castThis.removalListener = checkNotNull(listener); 164 | return castThis; 165 | } 166 | 167 | /** 168 | * Sets a location for persisting files. This directory must not be used for other purposes. 169 | * 170 | * @param persistenceDirectory A directory which is used by this file cache. 171 | * @return This builder. 172 | */ 173 | public FileSystemCacheBuilder persistenceDirectory(File persistenceDirectory) { 174 | checkState(this.persistenceDirectory == null); 175 | this.persistenceDirectory = checkNotNull(persistenceDirectory); 176 | return this; 177 | } 178 | 179 | /** 180 | * {@link com.google.common.cache.CacheBuilder#build()} 181 | */ 182 | public Cache build() { 183 | if (persistenceDirectory == null) { 184 | return new FileSystemPersistingCache(underlyingCacheBuilder, FileSystemCacheBuilder.castRemovalListener(removalListener)); 185 | } else { 186 | return new FileSystemPersistingCache(underlyingCacheBuilder, persistenceDirectory, FileSystemCacheBuilder.castRemovalListener(removalListener)); 187 | } 188 | } 189 | 190 | /** 191 | * {@link CacheBuilder#build(com.google.common.cache.CacheLoader)} 192 | */ 193 | public LoadingCache build(CacheLoader loader) { 194 | if (persistenceDirectory == null) { 195 | return new FileSystemLoadingPersistingCache(underlyingCacheBuilder, loader, FileSystemCacheBuilder.castRemovalListener(removalListener)); 196 | } else { 197 | return new FileSystemLoadingPersistingCache(underlyingCacheBuilder, loader, persistenceDirectory, FileSystemCacheBuilder.castRemovalListener(removalListener)); 198 | } 199 | } 200 | 201 | @SuppressWarnings("unchecked") 202 | private static RemovalListener castRemovalListener(RemovalListener removalListener) { 203 | if (removalListener == null) { 204 | return null; 205 | } else { 206 | return (RemovalListener) removalListener; 207 | } 208 | } 209 | 210 | @Override 211 | public String toString() { 212 | return "FileSystemCacheBuilder{" + 213 | "underlyingCacheBuilder=" + underlyingCacheBuilder + 214 | ", persistenceDirectory=" + persistenceDirectory + 215 | '}'; 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /src/main/java/com/blogspot/mydailyjava/guava/cache/overflow/FileSystemLoadingPersistingCache.java: -------------------------------------------------------------------------------- 1 | package com.blogspot.mydailyjava.guava.cache.overflow; 2 | 3 | import com.google.common.cache.CacheBuilder; 4 | import com.google.common.cache.CacheLoader; 5 | import com.google.common.cache.LoadingCache; 6 | import com.google.common.cache.RemovalListener; 7 | import com.google.common.collect.ImmutableMap; 8 | import com.google.common.io.Files; 9 | 10 | import java.io.File; 11 | import java.util.concurrent.Callable; 12 | import java.util.concurrent.ExecutionException; 13 | 14 | public class FileSystemLoadingPersistingCache extends FileSystemPersistingCache implements LoadingCache { 15 | 16 | private final CacheLoader cacheLoader; 17 | 18 | protected FileSystemLoadingPersistingCache(CacheBuilder cacheBuilder, CacheLoader cacheLoader) { 19 | this(cacheBuilder, cacheLoader, Files.createTempDir()); 20 | } 21 | 22 | protected FileSystemLoadingPersistingCache(CacheBuilder cacheBuilder, CacheLoader cacheLoader, File persistenceDirectory) { 23 | this(cacheBuilder, cacheLoader, persistenceDirectory, null); 24 | } 25 | 26 | protected FileSystemLoadingPersistingCache(CacheBuilder cacheBuilder, CacheLoader cacheLoader, RemovalListener removalListener) { 27 | this(cacheBuilder, cacheLoader, Files.createTempDir(), removalListener); 28 | } 29 | 30 | protected FileSystemLoadingPersistingCache(CacheBuilder cacheBuilder, CacheLoader cacheLoader, 31 | File persistenceDirectory, RemovalListener removalListener) { 32 | super(cacheBuilder, persistenceDirectory, removalListener); 33 | this.cacheLoader = cacheLoader; 34 | } 35 | 36 | private class ValueLoaderFromCacheLoader implements Callable { 37 | 38 | private final K key; 39 | private final CacheLoader cacheLoader; 40 | 41 | private ValueLoaderFromCacheLoader(CacheLoader cacheLoader, K key) { 42 | this.key = key; 43 | this.cacheLoader = cacheLoader; 44 | } 45 | 46 | @Override 47 | public V call() throws Exception { 48 | return cacheLoader.load(key); 49 | } 50 | } 51 | 52 | @Override 53 | public V get(K key) throws ExecutionException { 54 | return get(key, new ValueLoaderFromCacheLoader(cacheLoader, key)); 55 | } 56 | 57 | @Override 58 | public V getUnchecked(K key) { 59 | try { 60 | return get(key, new ValueLoaderFromCacheLoader(cacheLoader, key)); 61 | } catch (ExecutionException e) { 62 | throw new RuntimeException(e.getCause()); 63 | } 64 | } 65 | 66 | @Override 67 | public ImmutableMap getAll(Iterable keys) throws ExecutionException { 68 | ImmutableMap.Builder all = ImmutableMap.builder(); 69 | for (K key : keys) { 70 | all.put(key, get(key)); 71 | } 72 | return all.build(); 73 | } 74 | 75 | @Override 76 | public V apply(K key) { 77 | try { 78 | return cacheLoader.load(key); 79 | } catch (Exception e) { 80 | throw new RuntimeException(String.format("Could not apply cache on key %s", key), e); 81 | } 82 | } 83 | 84 | @Override 85 | public void refresh(K key) { 86 | try { 87 | getUnderlyingCache().put(key, cacheLoader.load(key)); 88 | } catch (Exception e) { 89 | throw new RuntimeException(String.format("Could not refresh value for key %s", key), e); 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/blogspot/mydailyjava/guava/cache/overflow/FileSystemPersistingCache.java: -------------------------------------------------------------------------------- 1 | package com.blogspot.mydailyjava.guava.cache.overflow; 2 | 3 | import com.google.common.cache.CacheBuilder; 4 | import com.google.common.cache.RemovalListener; 5 | import com.google.common.io.Files; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.io.*; 10 | import java.nio.channels.FileLock; 11 | import java.util.Arrays; 12 | import java.util.List; 13 | 14 | public class FileSystemPersistingCache extends AbstractPersistingCache { 15 | 16 | private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemPersistingCache.class); 17 | 18 | private final File persistenceRootDirectory; 19 | 20 | protected FileSystemPersistingCache(CacheBuilder cacheBuilder) { 21 | this(cacheBuilder, Files.createTempDir()); 22 | } 23 | 24 | protected FileSystemPersistingCache(CacheBuilder cacheBuilder, File persistenceDirectory) { 25 | this(cacheBuilder, persistenceDirectory, null); 26 | } 27 | 28 | protected FileSystemPersistingCache(CacheBuilder cacheBuilder, RemovalListener removalListener) { 29 | this(cacheBuilder, Files.createTempDir(), removalListener); 30 | } 31 | 32 | protected FileSystemPersistingCache(CacheBuilder cacheBuilder, File persistenceDirectory, RemovalListener removalListener) { 33 | super(cacheBuilder, removalListener); 34 | this.persistenceRootDirectory = validateDirectory(persistenceDirectory); 35 | LOGGER.info("Persisting to {}", persistenceDirectory.getAbsolutePath()); 36 | } 37 | 38 | private File validateDirectory(File directory) { 39 | directory.mkdirs(); 40 | if (!directory.exists() || !directory.isDirectory() || !directory.canRead() || !directory.canWrite()) { 41 | throw new IllegalArgumentException(String.format("Directory %s cannot be used as a persistence directory", 42 | directory.getAbsolutePath())); 43 | } 44 | return directory; 45 | } 46 | 47 | private File pathToFileFor(K key) { 48 | List pathSegments = directoryFor(key); 49 | File persistenceFile = persistenceRootDirectory; 50 | for (String pathSegment : pathSegments) { 51 | persistenceFile = new File(persistenceFile, pathSegment); 52 | } 53 | if (persistenceRootDirectory.equals(persistenceFile) || persistenceFile.isDirectory()) { 54 | throw new IllegalArgumentException(); 55 | } 56 | return persistenceFile; 57 | } 58 | 59 | @Override 60 | protected V findPersisted(K key) throws IOException { 61 | if (!isPersist(key)) return null; 62 | File persistenceFile = pathToFileFor(key); 63 | if (!persistenceFile.exists()) return null; 64 | FileInputStream fileInputStream = new FileInputStream(persistenceFile); 65 | try { 66 | FileLock fileLock = fileInputStream.getChannel().lock(0, Long.MAX_VALUE, true); 67 | try { 68 | return readPersisted(key, fileInputStream); 69 | } finally { 70 | fileLock.release(); 71 | } 72 | } catch (Exception e) { 73 | e.printStackTrace(); 74 | throw new RuntimeException(e); 75 | } finally { 76 | fileInputStream.close(); 77 | } 78 | } 79 | 80 | @Override 81 | protected void persistValue(K key, V value) throws IOException { 82 | if (!isPersist(key)) return; 83 | File persistenceFile = pathToFileFor(key); 84 | persistenceFile.getParentFile().mkdirs(); 85 | FileOutputStream fileOutputStream = new FileOutputStream(persistenceFile); 86 | try { 87 | FileLock fileLock = fileOutputStream.getChannel().lock(); 88 | try { 89 | persist(key, value, fileOutputStream); 90 | } finally { 91 | fileLock.release(); 92 | } 93 | } finally { 94 | fileOutputStream.close(); 95 | } 96 | } 97 | 98 | @Override 99 | protected void persist(K key, V value, OutputStream outputStream) throws IOException { 100 | ObjectOutputStream objectOutputStream = new ObjectOutputStream(outputStream); 101 | objectOutputStream.writeObject(value); 102 | objectOutputStream.flush(); 103 | } 104 | 105 | @Override 106 | protected boolean isPersist(K key) { 107 | return true; 108 | } 109 | 110 | @Override 111 | protected List directoryFor(K key) { 112 | return Arrays.asList(key.toString()); 113 | } 114 | 115 | @Override 116 | @SuppressWarnings("unchecked") 117 | protected V readPersisted(K key, InputStream inputStream) throws IOException { 118 | try { 119 | return (V) new ObjectInputStream(inputStream).readObject(); 120 | } catch (ClassNotFoundException e) { 121 | throw new RuntimeException(String.format("Serialized version assigned by %s was invalid", key), e); 122 | } 123 | } 124 | 125 | @Override 126 | protected void deletePersistedIfExistent(K key) { 127 | File file = pathToFileFor(key); 128 | file.delete(); 129 | } 130 | 131 | @Override 132 | protected void deleteAllPersisted() { 133 | for (File file : persistenceRootDirectory.listFiles()) { 134 | file.delete(); 135 | } 136 | } 137 | 138 | @Override 139 | protected int sizeOfPersisted() { 140 | return countFilesInFolders(persistenceRootDirectory); 141 | } 142 | 143 | private int countFilesInFolders(File directory) { 144 | int size = 0; 145 | for (File file : directory.listFiles()) { 146 | if (file.isDirectory()) { 147 | size += countFilesInFolders(file); 148 | } else if (!file.getName().startsWith(".")) { 149 | size++; 150 | } 151 | } 152 | return size; 153 | } 154 | 155 | public File getPersistenceRootDirectory() { 156 | return persistenceRootDirectory; 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /src/main/java/com/blogspot/mydailyjava/guava/cache/overflow/NotPersistedException.java: -------------------------------------------------------------------------------- 1 | package com.blogspot.mydailyjava.guava.cache.overflow; 2 | 3 | class NotPersistedException extends RuntimeException { 4 | /* empty */ 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/com/blogspot/mydailyjava/guava/cache/overflow/RemovalNotifications.java: -------------------------------------------------------------------------------- 1 | package com.blogspot.mydailyjava.guava.cache.overflow; 2 | 3 | import com.google.common.cache.RemovalCause; 4 | import com.google.common.cache.RemovalNotification; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.lang.reflect.Constructor; 9 | import java.lang.reflect.InvocationTargetException; 10 | 11 | class RemovalNotifications { 12 | 13 | private static final Logger LOGGER = LoggerFactory.getLogger(RemovalNotifications.class); 14 | 15 | private static RemovalNotifications INSTANCE; 16 | 17 | private static RemovalNotifications getInstance() { 18 | if (INSTANCE != null) return INSTANCE; 19 | synchronized (RemovalNotifications.class) { 20 | if (INSTANCE != null) return INSTANCE; 21 | return INSTANCE = new RemovalNotifications(); 22 | } 23 | } 24 | 25 | public static RemovalNotification make(K key, V value) { 26 | return getInstance().makeInternal(key, value); 27 | } 28 | 29 | private final Constructor constructor; 30 | 31 | private RemovalNotifications() { 32 | // Note: RemovalNotification constructor is package private 33 | try { 34 | constructor = RemovalNotification.class.getDeclaredConstructor(Object.class, Object.class, RemovalCause.class); 35 | constructor.setAccessible(true); 36 | } catch (NoSuchMethodException e) { 37 | String message = String.format("Could not find known constructor for %s", RemovalNotification.class.getCanonicalName()); 38 | LOGGER.error(message, e); 39 | throw new IllegalStateException(message, e); 40 | } 41 | } 42 | 43 | @SuppressWarnings("unchecked") 44 | private RemovalNotification makeInternal(K key, V value) { 45 | try { 46 | try { 47 | return (RemovalNotification) constructor.newInstance(key, value, RemovalCause.EXPLICIT); 48 | } catch (InvocationTargetException e) { 49 | throw new IllegalStateException(String.format("Creating an instance of %s for key %s and value %s caused an exception to be thrown", 50 | RemovalNotification.class.getCanonicalName(), key, value), e); 51 | } catch (InstantiationException e) { 52 | throw new IllegalStateException(String.format("Could not call %s's constructor for key %s and value %s", 53 | RemovalNotification.class.getCanonicalName(), key, value), e); 54 | } catch (IllegalAccessException e) { 55 | throw new IllegalStateException(String.format("Could not access %s's constructor for key %s and value %s", 56 | RemovalNotification.class.getCanonicalName(), key, value), e); 57 | } 58 | } catch (RuntimeException e) { 59 | LOGGER.error(e.getMessage(), e); 60 | throw e; 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/test/java/com/blogspot/mydailyjava/guava/cache/overflow/FileSystemLoadingPersistingCacheTest.java: -------------------------------------------------------------------------------- 1 | package com.blogspot.mydailyjava.guava.cache.overflow; 2 | 3 | import com.google.common.cache.CacheLoader; 4 | import com.google.common.cache.LoadingCache; 5 | import org.testng.annotations.AfterMethod; 6 | import org.testng.annotations.BeforeMethod; 7 | import org.testng.annotations.Test; 8 | 9 | import static org.testng.Assert.assertEquals; 10 | 11 | public class FileSystemLoadingPersistingCacheTest { 12 | 13 | private LoadingCache fileSystemPersistingCache; 14 | 15 | @BeforeMethod 16 | public void setUp() throws Exception { 17 | fileSystemPersistingCache = FileSystemCacheBuilder.newBuilder() 18 | .maximumSize(1L) 19 | .build(new CacheLoader() { 20 | @Override 21 | public String load(String key) throws Exception { 22 | return KeyValuePair.makeValue(KeyValuePair.fromKey(key)); 23 | } 24 | }); 25 | } 26 | 27 | @AfterMethod 28 | public void tearDown() throws Exception { 29 | fileSystemPersistingCache.invalidateAll(); 30 | } 31 | 32 | @Test 33 | public void testLoading() throws Exception { 34 | 35 | final int size = 100; 36 | 37 | for (int i = 0; i < size; i++) { 38 | assertEquals(fileSystemPersistingCache.getUnchecked(KeyValuePair.makeKey(i)), KeyValuePair.makeValue(i)); 39 | assertEquals(fileSystemPersistingCache.size(), i + 1); 40 | } 41 | 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/test/java/com/blogspot/mydailyjava/guava/cache/overflow/FileSystemPersistingCacheTest.java: -------------------------------------------------------------------------------- 1 | package com.blogspot.mydailyjava.guava.cache.overflow; 2 | 3 | import com.google.common.cache.Cache; 4 | import org.testng.annotations.AfterMethod; 5 | import org.testng.annotations.BeforeMethod; 6 | import org.testng.annotations.Test; 7 | 8 | import java.util.List; 9 | import java.util.concurrent.Callable; 10 | 11 | import static org.testng.Assert.*; 12 | 13 | public class FileSystemPersistingCacheTest { 14 | 15 | private Cache fileSystemPersistingCache; 16 | 17 | @BeforeMethod 18 | public void setUp() throws Exception { 19 | fileSystemPersistingCache = FileSystemCacheBuilder.newBuilder() 20 | .maximumSize(1L) 21 | .build(); 22 | } 23 | 24 | @AfterMethod 25 | public void tearDown() throws Exception { 26 | fileSystemPersistingCache.invalidateAll(); 27 | } 28 | 29 | @Test 30 | public void testCachePersistence() throws Exception { 31 | 32 | final int testSize = 100; 33 | List keyValuePairs = KeyValuePair.makeTestElements(testSize); 34 | 35 | for (KeyValuePair keyValuePair : keyValuePairs) { 36 | fileSystemPersistingCache.put(keyValuePair.getKey(), keyValuePair.getValue()); 37 | } 38 | 39 | assertEquals(fileSystemPersistingCache.size(), testSize); 40 | 41 | for (KeyValuePair keyValuePair : keyValuePairs) { 42 | String valueFromCache = fileSystemPersistingCache.getIfPresent(keyValuePair.getKey()); 43 | assertNotNull(valueFromCache); 44 | assertEquals(valueFromCache, keyValuePair.getValue()); 45 | assertEquals(fileSystemPersistingCache.size(), testSize); 46 | } 47 | 48 | final int manualDeleteSize = 10, factor = 3; 49 | assertTrue(manualDeleteSize * factor < testSize); 50 | 51 | for (int i = 0; i < manualDeleteSize; i++) { 52 | int index = i * factor; 53 | fileSystemPersistingCache.invalidate(KeyValuePair.makeKey(index)); 54 | String value = fileSystemPersistingCache.getIfPresent(KeyValuePair.makeKey(index)); 55 | assertNull(value); 56 | assertEquals(fileSystemPersistingCache.size(), testSize - i - 1); 57 | } 58 | 59 | fileSystemPersistingCache.invalidateAll(); 60 | assertEquals(fileSystemPersistingCache.size(), 0); 61 | } 62 | 63 | @Test 64 | public void testAddByCallable() throws Exception { 65 | 66 | final String callableReturnValue = "individual"; 67 | final Callable callable = new Callable() { 68 | @Override 69 | public String call() throws Exception { 70 | return callableReturnValue; 71 | } 72 | }; 73 | 74 | String value0 = fileSystemPersistingCache.get(KeyValuePair.makeKey(0), callable); 75 | assertNotNull(value0); 76 | assertEquals(value0, callableReturnValue); 77 | 78 | fileSystemPersistingCache.put(KeyValuePair.makeKey(1), KeyValuePair.makeValue(1)); 79 | String value1 = fileSystemPersistingCache.getIfPresent(KeyValuePair.makeKey(1)); 80 | assertNotNull(value1); 81 | assertEquals(value1, KeyValuePair.makeValue(1)); 82 | 83 | value1 = fileSystemPersistingCache.get(KeyValuePair.makeKey(1), callable); 84 | assertNotNull(value1); 85 | assertEquals(value1, KeyValuePair.makeValue(1)); 86 | 87 | value0 = fileSystemPersistingCache.getIfPresent(KeyValuePair.makeKey(0)); 88 | assertNotNull(value0); 89 | assertEquals(value0, callableReturnValue); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/test/java/com/blogspot/mydailyjava/guava/cache/overflow/KeyValuePair.java: -------------------------------------------------------------------------------- 1 | package com.blogspot.mydailyjava.guava.cache.overflow; 2 | 3 | import java.util.LinkedList; 4 | import java.util.List; 5 | 6 | public class KeyValuePair { 7 | 8 | private static final String KEY_PREFIX = "key"; 9 | private static final String VALUE_PREFIX = "value"; 10 | 11 | private final String key, value; 12 | 13 | public KeyValuePair(String key, String value) { 14 | this.key = key; 15 | this.value = value; 16 | } 17 | 18 | public String getKey() { 19 | return key; 20 | } 21 | 22 | public String getValue() { 23 | return value; 24 | } 25 | 26 | @Override 27 | public boolean equals(Object o) { 28 | if (this == o) return true; 29 | if (o == null || getClass() != o.getClass()) return false; 30 | 31 | KeyValuePair that = (KeyValuePair) o; 32 | 33 | return !(key != null ? !key.equals(that.key) : that.key != null) 34 | && !(value != null ? !value.equals(that.value) : that.value != null); 35 | 36 | } 37 | 38 | @Override 39 | public int hashCode() { 40 | int result = key != null ? key.hashCode() : 0; 41 | result = 31 * result + (value != null ? value.hashCode() : 0); 42 | return result; 43 | } 44 | 45 | @Override 46 | public String toString() { 47 | return "KeyValuePair{" + 48 | "key='" + key + '\'' + 49 | ", value='" + value + '\'' + 50 | '}'; 51 | } 52 | 53 | public static String makeKey(int i) { 54 | return KEY_PREFIX + i; 55 | } 56 | 57 | public static String makeValue(int i) { 58 | return VALUE_PREFIX + i; 59 | } 60 | 61 | public static int fromKey(String value) { 62 | return Integer.valueOf(value.substring(KEY_PREFIX.length(), value.length())); 63 | } 64 | 65 | public static int fromValue(String value) { 66 | return Integer.valueOf(value.substring(VALUE_PREFIX.length(), value.length())); 67 | } 68 | 69 | public static List makeTestElements(int size) { 70 | List testElements = new LinkedList(); 71 | for (int i = 0; i < size; i++) { 72 | testElements.add(new KeyValuePair(makeKey(i), makeValue(i))); 73 | } 74 | return testElements; 75 | } 76 | } -------------------------------------------------------------------------------- /src/test/java/com/blogspot/mydailyjava/guava/cache/overflow/RemovalListenerTest.java: -------------------------------------------------------------------------------- 1 | package com.blogspot.mydailyjava.guava.cache.overflow; 2 | 3 | import com.google.common.cache.Cache; 4 | import com.google.common.cache.RemovalListener; 5 | import com.google.common.cache.RemovalNotification; 6 | import org.testng.annotations.AfterMethod; 7 | import org.testng.annotations.BeforeMethod; 8 | import org.testng.annotations.Test; 9 | 10 | import static org.testng.Assert.assertTrue; 11 | 12 | public class RemovalListenerTest { 13 | 14 | private Cache fileSystemPersistingCache; 15 | private boolean[] listenerResults; 16 | 17 | @BeforeMethod 18 | public void setUp() throws Exception { 19 | listenerResults = new boolean[100]; 20 | fileSystemPersistingCache = FileSystemCacheBuilder.newBuilder() 21 | .maximumSize(1L) 22 | .removalListener(new RemovalListener() { 23 | @Override 24 | public void onRemoval(RemovalNotification notification) { 25 | listenerResults[KeyValuePair.fromKey(notification.getKey())] = true; 26 | } 27 | }) 28 | .build(); 29 | } 30 | 31 | @AfterMethod 32 | public void tearDown() throws Exception { 33 | fileSystemPersistingCache.invalidateAll(); 34 | } 35 | 36 | @Test 37 | public void testRemovalListener() throws Exception { 38 | 39 | for (int i = 0; i < listenerResults.length; i++) { 40 | fileSystemPersistingCache.put(KeyValuePair.makeKey(i), KeyValuePair.makeValue(i)); 41 | } 42 | 43 | for (int i = 0; i < listenerResults.length; i++) { 44 | fileSystemPersistingCache.invalidate(KeyValuePair.makeKey(i)); 45 | } 46 | 47 | for (boolean listenerResult : listenerResults) { 48 | assertTrue(listenerResult); 49 | } 50 | 51 | } 52 | } 53 | --------------------------------------------------------------------------------