├── .gitignore
├── src
    ├── main
    │   └── java
    │   │   └── com
    │   │       └── ibm
    │   │           └── cloud
    │   │               └── cache
    │   │                   └── redis
    │   │                       ├── ApplicationContextHolder.java
    │   │                       ├── CircuitBreakerRedisCacheAspect.java
    │   │                       └── CircuitBreakerRedisCache.java
    └── test
    │   └── java
    │       └── com
    │           └── ibm
    │               └── cloud
    │                   └── cache
    │                       └── redis
    │                           ├── CircuitBreakerRedisCacheManager.java
    │                           └── CircuitBreakerRedisCacheTest.java
├── pom.xml
└── README.md


/.gitignore:
--------------------------------------------------------------------------------
 1 | /target
 2 | /.m2
 3 | 
 4 | /.classpath
 5 | /.project
 6 | /.settings
 7 | 
 8 | .*.swp
 9 | .DS_Store
10 | 
11 | .idea
12 | *.iml
13 | 
14 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/cloud/cache/redis/ApplicationContextHolder.java:
--------------------------------------------------------------------------------
 1 | package com.ibm.cloud.cache.redis;
 2 | 
 3 | import org.springframework.beans.BeansException;
 4 | import org.springframework.context.ApplicationContext;
 5 | import org.springframework.context.ApplicationContextAware;
 6 | import org.springframework.stereotype.Component;
 7 | 
 8 | /**
 9 |  * Used to access Spring-instantiated beans.
10 |  */
11 | @Component
12 | public class ApplicationContextHolder implements ApplicationContextAware {
13 | 	
14 |     private static ApplicationContext context;
15 | 
16 |     @Override
17 |     public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
18 |         setAppCtx(applicationContext);   
19 |     }
20 |     
21 |     private static synchronized void setAppCtx(ApplicationContext applicationContext) {
22 |     	if (context == null) {
23 |     		context = applicationContext;
24 |     	}
25 |     }
26 | 
27 |     public static ApplicationContext getContext() {
28 |         return context;
29 |     }
30 | }


--------------------------------------------------------------------------------
/src/main/java/com/ibm/cloud/cache/redis/CircuitBreakerRedisCacheAspect.java:
--------------------------------------------------------------------------------
 1 | package com.ibm.cloud.cache.redis;
 2 | 
 3 | import java.util.logging.Logger;
 4 | 
 5 | import org.aspectj.lang.ProceedingJoinPoint;
 6 | import org.aspectj.lang.annotation.Around;
 7 | import org.aspectj.lang.annotation.Aspect;
 8 | import org.springframework.cache.Cache;
 9 | import org.springframework.data.redis.cache.RedisCache;
10 | import org.springframework.stereotype.Component;
11 | 
12 | /**
13 |  * Allows for modifying Redis cache instances to use a circuit breaker by utilizing Spring's Aspect Oriented Programming (AOP) support.
14 |  */
15 | @Aspect
16 | @Component
17 | public class CircuitBreakerRedisCacheAspect {
18 |     
19 |     private static final Logger logger = Logger.getLogger(CircuitBreakerRedisCacheAspect.class.getName());
20 | 
21 | 	public CircuitBreakerRedisCacheAspect() {
22 | 	}
23 | 	
24 | 	/**
25 | 	 * Wraps <code>RedisCache</code> instances with <code>CircuitBreakerRedisCache</code>. Every time a cache manager 
26 | 	 * returns a cache instance, this method checks if it is a <code>RedisCache</code> and if so, wraps it with our circuit breaker 
27 | 	 * implementation.
28 | 	 * 
29 | 	 * @return A <code>CircuitBreakerRedisCache</code> if the underlying cache instance is of type <code>RedisCache</code>; otherwise 
30 | 	 * returns the underlying cache instance.
31 | 	 */
32 | 	@Around("execution(* org.springframework.cache.support.AbstractCacheManager.getCache(..))")
33 |     public Cache beforeCacheGet(ProceedingJoinPoint proceedingJoinPoint) {
34 |         try {
35 |             Cache cache = (Cache) proceedingJoinPoint.proceed(proceedingJoinPoint.getArgs());
36 |             if (cache instanceof RedisCache) {
37 |             	logger.finest("Creating CircuitBreakerRedisCache");
38 |                 return new CircuitBreakerRedisCache((RedisCache) cache);
39 |             } else {
40 |                 return cache;
41 |             }
42 |         } catch (Throwable ex) {
43 |             return null;
44 |         }
45 |     }
46 | 
47 | }
48 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
 1 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 2 | 	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 3 | 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 4 | 	<modelVersion>4.0.0</modelVersion>
 5 | 	<groupId>com.ibm.cloud</groupId>
 6 | 	<artifactId>circuit-breaker-redis-cache</artifactId>
 7 | 	<version>1.0.0-SNAPSHOT</version>
 8 | 	<name>Circuit Breaker Redis Cache</name>
 9 | 	<description>A drop-in replacement for RedisCache with a circuit breaker</description>
10 | 
11 |     <properties>
12 |         <app.name>circuit-breaker-redis-cache</app.name>
13 |         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
14 |         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
15 |         <maven.compiler.source>1.8</maven.compiler.source>
16 |         <maven.compiler.target>1.8</maven.compiler.target>
17 |         <java.version>1.8</java.version>
18 |         
19 |         <spring.version>5.2.1.RELEASE</spring.version>
20 |     </properties>
21 | 
22 | 	<dependencies>
23 | 		<dependency>
24 | 			<groupId>org.springframework</groupId>
25 | 			<artifactId>spring-web</artifactId>
26 | 			<version>${spring.version}</version>
27 | 		</dependency>
28 |         <dependency>
29 |             <groupId>org.springframework</groupId>
30 |             <artifactId>spring-aspects</artifactId>
31 |             <version>${spring.version}</version>
32 |         </dependency>
33 | 		<dependency>
34 | 			<groupId>org.springframework.data</groupId>
35 | 			<artifactId>spring-data-redis</artifactId>
36 | 			<version>2.2.1.RELEASE</version>
37 | 		</dependency>
38 | 		<dependency>
39 | 			<groupId>io.github.resilience4j</groupId>
40 | 			<artifactId>resilience4j-circuitbreaker</artifactId>
41 | 			<version>1.1.0</version>
42 | 		</dependency>
43 | 
44 |         <!-- test -->
45 |         <dependency>
46 |             <groupId>junit</groupId>
47 |             <artifactId>junit</artifactId>
48 |             <type>jar</type>
49 |             <scope>test</scope>
50 |             <version>4.12</version>
51 |         </dependency>
52 |         <dependency>
53 |             <groupId>org.mockito</groupId>
54 |             <artifactId>mockito-core</artifactId>
55 |             <!-- releases greater than this break the unit tests -->
56 |             <version>2.10.0</version>
57 |         </dependency>
58 | 	</dependencies>
59 | </project>


--------------------------------------------------------------------------------
/src/test/java/com/ibm/cloud/cache/redis/CircuitBreakerRedisCacheManager.java:
--------------------------------------------------------------------------------
 1 | package com.ibm.cloud.cache.redis;
 2 | /**
 3 |  * 
 4 |  */
 5 | 
 6 | 
 7 | import java.util.Map;
 8 | 
 9 | import org.springframework.data.redis.cache.RedisCache;
10 | import org.springframework.data.redis.cache.RedisCacheConfiguration;
11 | import org.springframework.data.redis.cache.RedisCacheManager;
12 | import org.springframework.data.redis.cache.RedisCacheWriter;
13 | 
14 | /**
15 |  * Only needed to facilitate unit testing.
16 |  */
17 | public class CircuitBreakerRedisCacheManager extends RedisCacheManager {
18 |     
19 | 
20 |     /**
21 |      * @param cacheWriter
22 |      * @param defaultCacheConfiguration
23 |      */
24 |     public CircuitBreakerRedisCacheManager(RedisCacheWriter cacheWriter,
25 |             RedisCacheConfiguration defaultCacheConfiguration) {
26 |         super(cacheWriter, defaultCacheConfiguration);
27 |     }
28 | 
29 |     /**
30 |      * @param cacheWriter
31 |      * @param defaultCacheConfiguration
32 |      * @param initialCacheNames
33 |      */
34 |     public CircuitBreakerRedisCacheManager(RedisCacheWriter cacheWriter,
35 |             RedisCacheConfiguration defaultCacheConfiguration, String... initialCacheNames) {
36 |         super(cacheWriter, defaultCacheConfiguration, initialCacheNames);
37 |     }
38 | 
39 |     /**
40 |      * @param cacheWriter
41 |      * @param defaultCacheConfiguration
42 |      * @param initialCacheConfigurations
43 |      */
44 |     public CircuitBreakerRedisCacheManager(RedisCacheWriter cacheWriter,
45 |             RedisCacheConfiguration defaultCacheConfiguration,
46 |             Map<String, RedisCacheConfiguration> initialCacheConfigurations) {
47 |         super(cacheWriter, defaultCacheConfiguration, initialCacheConfigurations);
48 |     }
49 | 
50 |     /**
51 |      * @param cacheWriter
52 |      * @param defaultCacheConfiguration
53 |      * @param allowInFlightCacheCreation
54 |      * @param initialCacheNames
55 |      */
56 |     public CircuitBreakerRedisCacheManager(RedisCacheWriter cacheWriter,
57 |             RedisCacheConfiguration defaultCacheConfiguration, boolean allowInFlightCacheCreation,
58 |             String... initialCacheNames) {
59 |         super(cacheWriter, defaultCacheConfiguration, allowInFlightCacheCreation, initialCacheNames);
60 |     }
61 | 
62 |     /**
63 |      * @param cacheWriter
64 |      * @param defaultCacheConfiguration
65 |      * @param initialCacheConfigurations
66 |      * @param allowInFlightCacheCreation
67 |      */
68 |     public CircuitBreakerRedisCacheManager(RedisCacheWriter cacheWriter,
69 |             RedisCacheConfiguration defaultCacheConfiguration,
70 |             Map<String, RedisCacheConfiguration> initialCacheConfigurations, boolean allowInFlightCacheCreation) {
71 |         super(cacheWriter, defaultCacheConfiguration, initialCacheConfigurations, allowInFlightCacheCreation);
72 |     }
73 |     
74 |     @Override
75 |     protected RedisCache createRedisCache(String name, RedisCacheConfiguration cacheConfig) {
76 |         return new CircuitBreakerRedisCache(super.createRedisCache(name, cacheConfig));
77 |     }
78 |     
79 |     public RedisCache createBaseRedisCache(String name, RedisCacheConfiguration cacheConfig) {
80 |         return super.createRedisCache(name, cacheConfig);
81 |     }
82 | 
83 | }
84 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Circuit Breaker Redis Cache
  2 | This project provides a wrapper around Spring Data Redis `RedisCache` that incorporates a Circuit Breaker from the [resilience4j](https://github.com/resilience4j/resilience4j) project. This is useful in cases where the Redis server is down or slow and the application needs to continue servicing requests without a cache, albeit more slowly. In certain situations, Redis server instability coupled with Spring Data Redis Cache usage can lead to application instability. A circuit breaker provides an elegant solution in these scenarios.
  3 | 
  4 | The class `com.ibm.cloud.cache.redis.CircuitBreakerRedisCache` wraps all calls to the underlying `RedisCache` with `io.github.resilience4j.circuitbreaker.CircuitBreaker` decoration such that if the calls fail enough to open the circuit, the calls will be subsequently bypassed until the circuit closes. 
  5 | 
  6 | ## How to use
  7 | The wrapping is implemented via [Spring Aspects](https://docs.spring.io/spring/docs/current/spring-framework-reference/core.html#aop-atconfigurable) such that whenever a `RedisCache` instance is requested from the cache manager, it is wrapped by `CircuitBreakerRedisCache`. Therefore an application only needs the following to make use of this cache:
  8 | 1. Add this project as a dependency.
  9 | 2. Create `@Bean`-annotated methods that will create the following objects: `io.github.resilience4j.circuitbreaker.CircuitBreakerConfig`, `io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry`, `io.github.resilience4j.circuitbreaker.CircuitBreaker`. These methods typically reside in an extension of `org.springframework.cache.annotation.CachingConfigurerSupport` when utilizing [Spring Cache](https://spring.io/guides/gs/caching/).
 10 | 
 11 | ## Sample configuration code
 12 | Example Circuit Breaker configuration beans follow. With this code:
 13 | - The Circuit Breaker sliding window is configured to be time-based, with a duration of 2 minutes. 
 14 | - The failure and slow call rates are both set to 50%.
 15 | - A minimum of 10 calls must be made before the circuit breaker can begin opening.
 16 | 
 17 | If 50% of the calls within the sliding window period fail or are slower than 1.5 seconds, the circuit is open and `RedisCache` calls are not made. At this point `CircuitBreakerRedisCache` behaves as a no-op cache, triggering cache misses. After 3 minutes, the circuit goes to `HALF_OPEN` state, allowing up to 50 `RedisCache` calls to see whether they succeed. If at least 50% calls do succeed, then the circuit is closed and `RedisCache` calls are made normally.
 18 | 
 19 | ```java
 20 | // ...
 21 | import java.time.Duration;
 22 | import java.util.Optional;
 23 | import java.util.logging.Level;
 24 | import java.util.logging.Logger;
 25 | import org.springframework.context.annotation.Bean;
 26 | import org.springframework.context.annotation.Lazy;
 27 | import io.github.resilience4j.circuitbreaker.CircuitBreaker;
 28 | import io.github.resilience4j.circuitbreaker.CircuitBreaker.Metrics;
 29 | import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
 30 | import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig.SlidingWindowType;
 31 | import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry;
 32 | import io.github.resilience4j.circuitbreaker.event.CircuitBreakerOnStateTransitionEvent;
 33 | 
 34 | // ...
 35 | 
 36 | public static final String CB_NAME = "myAppCB";
 37 | public static final String CB_CONFIG_NAME = "myAppCBConfig";
 38 | public static final int CB_SLOW_RATE_THRESHOLD = 50;
 39 | public static final int CB_FAILURE_RATE_THRESHOLD = 50;
 40 | public static final Duration CB_SLOW_CALL_DURATION = Duration.ofMillis(1500); // 1.5 secs
 41 | public static final int CB_NUM_CALLS_HALF_OPEN_STATE = 50;
 42 | public static final int CB_MIN_NUM_CALLS = 10;
 43 | public static final Duration CB_WAIT_DURATION_OPEN_STATE = Duration.ofMillis(3*60*1000); // 3 minutes
 44 | public static final int CB_SLIDING_WINDOW_SIZE = 60*2; // 2 minutes
 45 | 
 46 | @Bean
 47 | @Lazy
 48 | public CircuitBreaker defaultCircuitBreaker(CircuitBreakerRegistry circuitBreakerRegistry) {
 49 |     CircuitBreaker circuitBreaker = circuitBreakerRegistry.circuitBreaker(CB_NAME, CB_CONFIG_NAME);
 50 |     circuitBreaker.getEventPublisher().onStateTransition(event -> this.circuitBreakerStateTransition(event, circuitBreaker));
 51 |     return circuitBreaker;
 52 | }
 53 | 
 54 | private void circuitBreakerStateTransition(CircuitBreakerOnStateTransitionEvent event, CircuitBreaker cb) {
 55 |     logger.warning("CircuitBreaker " + event.getCircuitBreakerName() 
 56 |       + " transitioned from " + event.getStateTransition().getFromState() 
 57 |       + " to " + event.getStateTransition().getToState());
 58 |     logCBMetrics(cb, Level.WARNING);
 59 | }
 60 | 
 61 | public static void logCBMetrics(CircuitBreaker cb, Level loggerLevel) {
 62 |     Metrics cbMetrics = cb.getMetrics();
 63 |     StringBuilder sb = new StringBuilder("CircuitBreaker Metrics for " + cb + ":\n");
 64 |     sb.append("\tNum successful calls: ").append(cbMetrics.getNumberOfSuccessfulCalls()).append("\n");
 65 |     sb.append("\tNum failed calls: ").append(cbMetrics.getNumberOfFailedCalls()).append("\n");
 66 |     sb.append("\tNum slow calls: ").append(cbMetrics.getNumberOfSlowCalls()).append("\n");
 67 |     sb.append("\tSlow call rate: ").append(cbMetrics.getSlowCallRate()).append("\n");
 68 |     sb.append("\tNum not permitted calls: ").append(cbMetrics.getNumberOfNotPermittedCalls()).append("\n");
 69 |     logger.log(loggerLevel, sb.toString());
 70 | }
 71 | 
 72 | @Bean
 73 | @Lazy
 74 | public CircuitBreakerRegistry defaultCircuitBreakerRegistry(CircuitBreakerConfig builtCBC) {
 75 |     CircuitBreakerRegistry cbr = CircuitBreakerRegistry.ofDefaults();
 76 |     CircuitBreakerConfig cbc;
 77 |     Optional<CircuitBreakerConfig> cbco = cbr.getConfiguration(CB_CONFIG_NAME);
 78 |     if (!cbco.isPresent()) {
 79 |       cbc = builtCBC;
 80 |       cbr.addConfiguration(CB_CONFIG_NAME, cbc);
 81 |     }
 82 |     return cbr;
 83 | }
 84 | 
 85 | @Bean
 86 | @Lazy
 87 | public CircuitBreakerConfig defaultCircuitBreakerConfig() {
 88 |     CircuitBreakerConfig cbc = CircuitBreakerConfig.custom()
 89 |           .failureRateThreshold(CB_FAILURE_RATE_THRESHOLD)
 90 |           .slowCallRateThreshold(CB_SLOW_RATE_THRESHOLD)
 91 |           .waitDurationInOpenState(CB_WAIT_DURATION_OPEN_STATE)
 92 |           .slowCallDurationThreshold(CB_SLOW_CALL_DURATION)
 93 |           .permittedNumberOfCallsInHalfOpenState(CB_NUM_CALLS_HALF_OPEN_STATE)
 94 |           .minimumNumberOfCalls(CB_MIN_NUM_CALLS)
 95 |           .slidingWindowType(SlidingWindowType.TIME_BASED)
 96 |           .slidingWindowSize(CB_SLIDING_WINDOW_SIZE)
 97 |           .build();
 98 |     return cbc;
 99 | }
100 | ```
101 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/cloud/cache/redis/CircuitBreakerRedisCache.java:
--------------------------------------------------------------------------------
  1 | package com.ibm.cloud.cache.redis;
  2 | 
  3 | import java.util.concurrent.Callable;
  4 | import java.util.logging.Level;
  5 | import java.util.logging.Logger;
  6 | 
  7 | import org.springframework.context.ApplicationContext;
  8 | import org.springframework.data.redis.cache.RedisCache;
  9 | 
 10 | import io.github.resilience4j.circuitbreaker.CallNotPermittedException;
 11 | import io.github.resilience4j.circuitbreaker.CircuitBreaker;
 12 | import io.github.resilience4j.circuitbreaker.CircuitBreaker.Metrics;
 13 | import io.vavr.CheckedFunction0;
 14 | import io.vavr.control.Try;
 15 | 
 16 | /**
 17 |  * <p>RedisCache implementation that uses a resilience4j Circuit Breaker to bypass the Redis instance calls if the circuit 
 18 |  * is open. This allows us to not be sensitive to Redis instabilities. If the call to Redis fails, Spring Cache 
 19 |  * will treat it as a cache miss and continue executing the cached method normally.
 20 |  */
 21 | public class CircuitBreakerRedisCache extends RedisCache {
 22 |     
 23 |     private static final Logger logger = Logger.getLogger(CircuitBreakerRedisCache.class.getName());
 24 |     private static final Level loggerLevel = Level.FINER;
 25 |     
 26 |     private RedisCache redisCache;
 27 |     private CircuitBreaker circuitBreaker;
 28 |     
 29 |     /**
 30 |      * 
 31 |      * @param redisCache The <code>RedisCache</code> instance to be wrapped.
 32 |      */
 33 |     public CircuitBreakerRedisCache(RedisCache redisCache) {
 34 |     	/*
 35 |     	 * this class is implemented using both composition and inheritance due to limitations in RedisCache that prevent 
 36 |     	 * properly extending it. if this PR is approved, composition is no longer needed: https://github.com/spring-projects/spring-data-redis/pull/500
 37 |     	 */
 38 |         super(redisCache.getName(), redisCache.getNativeCache(), redisCache.getCacheConfiguration());
 39 |         this.redisCache = redisCache;
 40 |         ApplicationContext appCtx = ApplicationContextHolder.getContext();
 41 |         if (appCtx != null) {
 42 |         	/*
 43 |         	 * using this bean retrieval instead of @Autowired due to the RedisCache field which needs to be passed and can't 
 44 |         	 * be instantiated directly.
 45 |         	 * 
 46 |         	 * this call can be changed to specify a bean qualifier name in cases where multiple circuit breakers are used.
 47 |         	 */
 48 |         	this.circuitBreaker = appCtx.getBean(CircuitBreaker.class);
 49 |         }
 50 |     }
 51 |     
 52 |     /**
 53 |      * Used by unit tests.
 54 |      * @param redisCache
 55 |      * @param cb
 56 |      */
 57 |     public CircuitBreakerRedisCache(RedisCache redisCache, CircuitBreaker cb) {
 58 |         this(redisCache);
 59 |         this.circuitBreaker = cb;
 60 |     }
 61 |     
 62 |     @Override
 63 |     public ValueWrapper get(Object key) {
 64 |     	this.logCBMetrics();
 65 |         CheckedFunction0<ValueWrapper> checkedFunction = this.circuitBreaker.decorateCheckedSupplier(() -> this.redisCache.get(key));
 66 |         Try<ValueWrapper> result = Try.of(checkedFunction).recoverWith(CallNotPermittedException.class, e -> this.recover(e));
 67 |         return result.get();
 68 |     }
 69 | 
 70 |     /**
 71 |      * Logs a portion of the Circuit Breaker's metrics at a predefined logger level.
 72 |      */
 73 | 	protected void logCBMetrics() {
 74 | 		logCBMetrics(this.circuitBreaker, loggerLevel);
 75 | 	}
 76 |     
 77 | 	/**
 78 | 	 * Subclasses can override to define new behavior when the circuit is open.
 79 | 	 * 
 80 | 	 * @param <T> The return type.
 81 | 	 * @param e A <code>CallNotPermittedException</code> exception indicating the call was not allowed because the circuit is open.
 82 | 	 * 
 83 | 	 * @return A <code>Success</code> with <code>null</code> return value.
 84 | 	 */
 85 |     protected <T> Try<T> recover(CallNotPermittedException e) {
 86 |     	logger.log(loggerLevel, "Circuit broken, cache bypassed: " + e.getMessage());
 87 |     	return Try.success(null);
 88 |     }
 89 |     
 90 |     /**
 91 |      * Subclasses can override to define new behavior when the circuit is open.
 92 |      * 
 93 | 	 * @param e A <code>CallNotPermittedException</code> exception indicating the call was not allowed because the circuit is open.
 94 |      * 
 95 |      * @return <code>null</code>
 96 |      */
 97 |     protected Void recoverVoid(CallNotPermittedException e) {
 98 |     	logger.log(loggerLevel, "Circuit broken, cache bypassed: " + e.getMessage());
 99 |     	return null;
100 |     }
101 |     
102 |     @Override
103 |     public <T> T get(Object key, Callable<T> valueLoader) {
104 |     	this.logCBMetrics();
105 |         CheckedFunction0<T> checkedFunction = this.circuitBreaker.decorateCheckedSupplier(() -> this.redisCache.get(key, valueLoader));
106 |         Try<T> result = Try.of(checkedFunction).recoverWith(CallNotPermittedException.class, e -> this.recover(e));
107 |         if (result.isFailure()) {
108 |             Throwable cause = result.getCause();
109 |             logger.log(loggerLevel, "Failure from RedisCache.get(): " + cause.getMessage());
110 |         }
111 |         return result.get();
112 |     }
113 |     
114 |     @Override
115 |     public <T> T get(Object key, Class<T> type) {
116 |     	this.logCBMetrics();
117 |         CheckedFunction0<T> checkedFunction = this.circuitBreaker.decorateCheckedSupplier(() -> this.redisCache.get(key, type));
118 |         Try<T> result = Try.of(checkedFunction).recoverWith(CallNotPermittedException.class, e -> this.recover(e));
119 |         if (result.isFailure()) {
120 |             Throwable cause = result.getCause();
121 |             logger.log(loggerLevel, "Failure from RedisCache.get(): " + cause.getMessage());
122 |         }
123 |         return result.get();
124 |     }
125 |     
126 |     @Override
127 |     public void put(Object key, Object value) {
128 |     	this.logCBMetrics();
129 |         Runnable decorateRunnable = this.circuitBreaker.decorateRunnable(() -> this.redisCache.put(key, value));
130 |         Try<Void> result = Try.runRunnable(decorateRunnable).recover(CallNotPermittedException.class, e -> this.recoverVoid(e));
131 |         if (result.isFailure()) {
132 |             Throwable cause = result.getCause();
133 |             logger.log(loggerLevel, "Failure from RedisCache.put(): " + cause.getMessage());
134 |         }
135 |         result.get();
136 |     }
137 |     
138 |     @Override
139 |     public ValueWrapper putIfAbsent(Object key, Object value) {
140 |     	this.logCBMetrics();
141 |         CheckedFunction0<ValueWrapper> checkedFunction = this.circuitBreaker.decorateCheckedSupplier(() -> this.redisCache.putIfAbsent(key, value));
142 |         Try<ValueWrapper> result = Try.of(checkedFunction).recoverWith(CallNotPermittedException.class, e -> this.recover(e));
143 |         if (result.isFailure()) {
144 |             Throwable cause = result.getCause();
145 |             logger.log(loggerLevel, "Failure from RedisCache.putIfAbsent(): " + cause.getMessage());
146 |         }
147 |         return result.get();
148 |     }
149 |     
150 |     @Override
151 |     public void evict(Object key) {
152 |     	this.logCBMetrics();
153 |         Runnable decorateRunnable = this.circuitBreaker.decorateRunnable(() -> this.redisCache.evict(key));
154 |         Try<Void> result = Try.runRunnable(decorateRunnable).recover(CallNotPermittedException.class, e -> this.recoverVoid(e));
155 |         if (result.isFailure()) {
156 |             Throwable cause = result.getCause();
157 |             logger.log(loggerLevel, "Failure from RedisCache.evict(): " + cause.getMessage());
158 |         }
159 |         result.get();
160 |     }
161 |     
162 |     @Override
163 |     public void clear() {
164 |     	this.logCBMetrics();
165 |         Runnable decorateRunnable = this.circuitBreaker.decorateRunnable(() -> this.redisCache.clear());
166 |         Try<Void> result = Try.runRunnable(decorateRunnable).recover(CallNotPermittedException.class, e -> this.recoverVoid(e));
167 |         if (result.isFailure()) {
168 |             Throwable cause = result.getCause();
169 |             logger.log(loggerLevel, "Failure from RedisCache.clear(): " + cause.getMessage());
170 |         }
171 |         result.get();
172 |     }
173 | 
174 | 	public static void logCBMetrics(CircuitBreaker cb, Level loggerLevel) {
175 | 		Metrics cbMetrics = cb.getMetrics();
176 |     	StringBuilder sb = new StringBuilder("CircuitBreaker Metrics for " + cb + ":\n");
177 |     	sb.append("\tNum successful calls: ").append(cbMetrics.getNumberOfSuccessfulCalls()).append("\n");
178 |     	sb.append("\tNum failed calls: ").append(cbMetrics.getNumberOfFailedCalls()).append("\n");
179 |     	sb.append("\tNum slow calls: ").append(cbMetrics.getNumberOfSlowCalls()).append("\n");
180 |     	sb.append("\tSlow call rate: ").append(cbMetrics.getSlowCallRate()).append("\n");
181 |     	sb.append("\tNum not permitted calls: ").append(cbMetrics.getNumberOfNotPermittedCalls()).append("\n");
182 |     	logger.log(loggerLevel, sb.toString());
183 | 	}
184 | 
185 | }
186 | 


--------------------------------------------------------------------------------
/src/test/java/com/ibm/cloud/cache/redis/CircuitBreakerRedisCacheTest.java:
--------------------------------------------------------------------------------
  1 | package com.ibm.cloud.cache.redis;
  2 | 
  3 | 
  4 | import static org.junit.Assert.assertEquals;
  5 | import static org.junit.Assert.assertNotNull;
  6 | import static org.junit.Assert.assertNull;
  7 | import static org.junit.Assert.fail;
  8 | 
  9 | import java.io.IOException;
 10 | import java.nio.ByteBuffer;
 11 | import java.time.Duration;
 12 | import java.util.Optional;
 13 | import java.util.logging.Level;
 14 | import java.util.logging.Logger;
 15 | 
 16 | import org.junit.Before;
 17 | import org.junit.Test;
 18 | import org.mockito.Mock;
 19 | import org.mockito.Mockito;
 20 | import org.mockito.MockitoAnnotations;
 21 | import org.mockito.invocation.InvocationOnMock;
 22 | import org.mockito.stubbing.Answer;
 23 | import org.springframework.cache.Cache.ValueWrapper;
 24 | import org.springframework.cache.support.SimpleValueWrapper;
 25 | import org.springframework.core.convert.ConversionService;
 26 | import org.springframework.core.convert.TypeDescriptor;
 27 | import org.springframework.data.redis.cache.RedisCache;
 28 | import org.springframework.data.redis.cache.RedisCacheConfiguration;
 29 | import org.springframework.data.redis.cache.RedisCacheWriter;
 30 | import org.springframework.data.redis.serializer.RedisSerializationContext.SerializationPair;
 31 | 
 32 | import io.github.resilience4j.circuitbreaker.CircuitBreaker;
 33 | import io.github.resilience4j.circuitbreaker.CircuitBreaker.State;
 34 | import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
 35 | import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig.SlidingWindowType;
 36 | import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry;
 37 | import io.github.resilience4j.circuitbreaker.event.CircuitBreakerOnStateTransitionEvent;
 38 | 
 39 | public class CircuitBreakerRedisCacheTest {
 40 |     
 41 |     private static final Logger logger = Logger.getLogger(CircuitBreakerRedisCacheTest.class.getName());
 42 |     
 43 |     private static final String TEST_KEY = "test";
 44 |     
 45 |     public static final String CB_NAME = "testRedisCB";
 46 |     public static final String CB_CONFIG_NAME = "testRedisCBConfig";
 47 |     public static final int CB_FAILURE_RATE_THRESHOLD = 100;
 48 |     public static final int CB_SLIDING_WINDOW_SIZE = 6;
 49 |     public static final int CB_MIN_NUM_CALLS = 6;
 50 |     public static final Duration CB_WAIT_DURATION_OPEN_STATE = Duration.ofSeconds(3);
 51 |     public static final int CB_NUM_CALLS_HALF_OPEN_STATE = 6;
 52 |     public static final Duration CB_SLOW_CALL_DURATION = Duration.ofMillis(1500); // 1.5 secs
 53 |     public static final int CB_SLOW_RATE_THRESHOLD = 50;
 54 |     
 55 |     @Mock
 56 |     private RedisCacheWriter cacheWriter;
 57 |     @Mock
 58 |     private RedisCacheConfiguration redisCacheConfig;
 59 |     @Mock
 60 |     private ConversionService conversionService;
 61 |     @Mock
 62 |     private SerializationPair<String> keySerializationPair;
 63 |     @Mock
 64 |     private SerializationPair<Object> valueSerializationPair;
 65 |     
 66 |     private RedisCache redisCache;
 67 |     
 68 |     private CircuitBreakerRedisCacheManager cacheMgr;
 69 |     
 70 |     private CircuitBreakerRedisCache cache;
 71 |     
 72 |     private CircuitBreaker cb;
 73 | 
 74 |     public CircuitBreakerConfig testCircuitBreakerConfig() {
 75 |         CircuitBreakerConfig cbc = CircuitBreakerConfig.custom()
 76 |                 .failureRateThreshold(CB_FAILURE_RATE_THRESHOLD)
 77 |                 .slowCallRateThreshold(CB_SLOW_RATE_THRESHOLD)
 78 |                 .waitDurationInOpenState(CB_WAIT_DURATION_OPEN_STATE)
 79 |                 .slowCallDurationThreshold(CB_SLOW_CALL_DURATION)
 80 |                 .permittedNumberOfCallsInHalfOpenState(CB_NUM_CALLS_HALF_OPEN_STATE)
 81 |                 .minimumNumberOfCalls(CB_MIN_NUM_CALLS)
 82 |                 .slidingWindowType(SlidingWindowType.COUNT_BASED)
 83 |                 .slidingWindowSize(CB_SLIDING_WINDOW_SIZE) // how many calls to collect in a single aggregation
 84 |                 .build();
 85 |         return cbc;
 86 |     }
 87 |     
 88 |     public CircuitBreaker defaultCircuitBreaker(CircuitBreakerRegistry circuitBreakerRegistry) {
 89 |         CircuitBreaker circuitBreaker = circuitBreakerRegistry.circuitBreaker(CB_NAME, CB_CONFIG_NAME);
 90 |         circuitBreaker.getEventPublisher().onStateTransition(event -> this.circuitBreakerStateTransition(event, circuitBreaker));
 91 |         return circuitBreaker;
 92 |     }
 93 |     
 94 |     private void circuitBreakerStateTransition(CircuitBreakerOnStateTransitionEvent event, CircuitBreaker cb) {
 95 |     	logger.warning("CircuitBreaker " + event.getCircuitBreakerName() 
 96 | 	        + " transitioned from " + event.getStateTransition().getFromState() 
 97 | 	        + " to " + event.getStateTransition().getToState());
 98 |     	CircuitBreakerRedisCache.logCBMetrics(cb, Level.WARNING);
 99 |     }
100 | 
101 |     public CircuitBreakerRegistry defaultCircuitBreakerRegistry(CircuitBreakerConfig builtCBC) {
102 |         CircuitBreakerRegistry cbr = CircuitBreakerRegistry.ofDefaults();
103 |         CircuitBreakerConfig cbc;
104 |         Optional<CircuitBreakerConfig> cbco = cbr.getConfiguration(CB_CONFIG_NAME);
105 |         if (!cbco.isPresent()) {
106 |             cbc = builtCBC;
107 |             cbr.addConfiguration(CB_CONFIG_NAME, cbc);
108 |         }
109 |         return cbr;
110 |     }
111 |     
112 |     @Before
113 |     public void setUp() {
114 |         MockitoAnnotations.initMocks(this);
115 |         Mockito.when(this.redisCacheConfig.getConversionService()).thenReturn(this.conversionService);
116 |         Mockito.when(this.redisCacheConfig.getKeySerializationPair()).thenReturn(this.keySerializationPair);
117 |         Mockito.when(this.redisCacheConfig.getValueSerializationPair()).thenReturn(this.valueSerializationPair);
118 |         Mockito.when(this.conversionService.canConvert(Mockito.any(TypeDescriptor.class), Mockito.any(TypeDescriptor.class))).thenReturn(false);
119 |         
120 |         this.cacheMgr = new CircuitBreakerRedisCacheManager(this.cacheWriter, this.redisCacheConfig);
121 |         this.redisCache = this.cacheMgr.createBaseRedisCache("testCBRedisCache", this.redisCacheConfig);
122 |         
123 |         CircuitBreakerConfig cbc = this.testCircuitBreakerConfig();
124 |         CircuitBreakerRegistry cbr = this.defaultCircuitBreakerRegistry(cbc);
125 |         this.cb = this.defaultCircuitBreaker(cbr);
126 |         this.cache = new CircuitBreakerRedisCache(this.redisCache, this.cb);
127 |     }
128 |     
129 |     @SuppressWarnings("unchecked")
130 |     protected void setUpForFailure() {
131 |         Mockito.when(this.cacheWriter.get(Mockito.anyString(), Mockito.any())).thenThrow(IOException.class);
132 |         Mockito.when(this.conversionService.canConvert(Mockito.any(TypeDescriptor.class), Mockito.any(TypeDescriptor.class))).thenReturn(false);
133 |         Mockito.when(this.conversionService.convert(Mockito.any(), Mockito.any(Class.class))).thenThrow(IOException.class);
134 |         Mockito.when(this.keySerializationPair.write(Mockito.anyString())).thenThrow(IOException.class);
135 |     }
136 |     
137 |     protected void setUpForSuccess() {
138 |         Mockito.reset(this.cacheWriter, this.keySerializationPair);
139 |         Mockito.when(this.cacheWriter.get(Mockito.anyString(), Mockito.any())).thenReturn(TEST_KEY.getBytes());
140 |         Mockito.when(this.cacheWriter.putIfAbsent(Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(TEST_KEY.getBytes());
141 |         Mockito.when(this.keySerializationPair.write(Mockito.anyString())).thenReturn(ByteBuffer.wrap(TEST_KEY.getBytes()));
142 |         Mockito.when(this.keySerializationPair.read(Mockito.any(ByteBuffer.class))).thenReturn(TEST_KEY);
143 |         Mockito.when(this.valueSerializationPair.write(Mockito.anyString())).thenReturn(ByteBuffer.wrap(TEST_KEY.getBytes()));
144 |         Mockito.when(this.valueSerializationPair.read(Mockito.any(ByteBuffer.class))).thenReturn(TEST_KEY);
145 |         Mockito.when(this.conversionService.convert(Mockito.any(), Mockito.eq(byte[].class))).thenReturn(TEST_KEY.getBytes());
146 |     }
147 |     
148 |     protected void setUpForSlowFailure() {
149 |         this.setUpForSuccess();
150 |         Mockito.when(this.cacheWriter.get(Mockito.anyString(), Mockito.any())).thenAnswer(new Answer<byte[]>() {
151 | 
152 | 			@Override
153 | 			public byte[] answer(InvocationOnMock invocation) throws Throwable {
154 | 				return slowReturn(TEST_KEY.getBytes());
155 | 			}
156 |         	
157 |         });
158 |     }
159 |     
160 |     public static <T> T slowReturn(T val) {
161 |     	long sleep = CB_SLOW_CALL_DURATION.toMillis() + 500;
162 |     	return slowReturn(val, sleep);
163 |     }
164 |     
165 |     public static <T> T slowReturn(T val, long sleep) {
166 |     	logger.info("slowReturn for " + sleep + " ms");
167 |     	try {
168 | 			Thread.sleep(sleep);
169 | 		} catch (InterruptedException e) {
170 | 		}
171 | 		return val;
172 |     }
173 | 
174 |     @Test
175 |     public void testCircuitBreakerOpened() {
176 |         this.setUpForFailure();
177 | 
178 |         for (int i = 0; i < CB_SLIDING_WINDOW_SIZE; i++) {
179 |             if (this.cb.getState().equals(State.CLOSED)) {
180 |                 try {
181 |                     this.cache.get(TEST_KEY);
182 |                     fail("Expected exception from redis cache");
183 |                 } catch (Exception e) {
184 |                     System.out.println("Exception from cache get: " + e.getMessage());
185 |                 }
186 |                 try {
187 |                     this.cache.get(TEST_KEY, () -> TEST_KEY);
188 |                     fail("Expected exception from redis cache");
189 |                 } catch (Exception e) {
190 |                     System.out.println("Exception from cache get: " + e.getMessage());
191 |                 }
192 |                 try {
193 |                     this.cache.put(TEST_KEY, TEST_KEY);
194 |                     fail("Expected exception from redis cache");
195 |                 } catch (Exception e) {
196 |                     System.out.println("Exception from cache put: " + e.getMessage());
197 |                 }
198 |                 try {
199 |                     this.cache.putIfAbsent(TEST_KEY, TEST_KEY);
200 |                     fail("Expected exception from redis cache");
201 |                 } catch (Exception e) {
202 |                     System.out.println("Exception from cache put: " + e.getMessage());
203 |                 }
204 |                 try {
205 |                     this.cache.evict(TEST_KEY);
206 |                     fail("Expected exception from redis cache");
207 |                 } catch (Exception e) {
208 |                     System.out.println("Exception from cache evict: " + e.getMessage());
209 |                 }
210 |                 try {
211 |                     this.cache.clear();
212 |                     fail("Expected exception from redis cache");
213 |                 } catch (Exception e) {
214 |                     System.out.println("Exception from cache clear: " + e.getMessage());
215 |                 }
216 |             }
217 |             
218 |             // circuit should be open now
219 |             assertEquals("Unexpected circuit breaker state", CircuitBreaker.State.OPEN, this.cb.getState());
220 |             try {
221 |                 ValueWrapper value = this.cache.get(TEST_KEY);
222 |                 assertEquals("Unexpected value", null, value);
223 |             } catch (Exception e) {
224 |                 e.printStackTrace();
225 |                 fail("Unexpected exception from redis cache: " + e.getMessage());
226 |             }
227 | 
228 |             try {
229 |                 this.cache.clear();
230 |             } catch (Exception e) {
231 |                 e.printStackTrace();
232 |                 fail("Unexpected exception from redis cache: " + e.getMessage());
233 |             }
234 |         }
235 |     }
236 | 
237 |     @Test
238 |     public void testCircuitBreakerClosed() {
239 |         this.setUpForFailure();
240 |         
241 |         for (int i = 0; i < CB_SLIDING_WINDOW_SIZE; i++) {
242 |             try {
243 |                 this.cache.get(TEST_KEY);
244 |                 fail("Expected exception from redis cache");
245 |             } catch (Exception e) {
246 |                 System.out.println("Exception from cache get: " + e.getMessage());
247 |             }
248 |         }
249 |         
250 |         // circuit should be open now
251 |         for (int i = 0; i < CB_SLIDING_WINDOW_SIZE; i++) {
252 |             try {
253 |                 ValueWrapper value = this.cache.get(TEST_KEY);
254 |                 assertEquals("Unexpected value", null, value);
255 |             } catch (Exception e) {
256 |                 e.printStackTrace();
257 |                 fail("Unexpected exception from redis cache: " + e.getMessage());
258 |             }
259 |             
260 |             assertEquals("Unexpected circuit breaker state", CircuitBreaker.State.OPEN, this.cb.getState());
261 |         }
262 |         
263 |         this.setUpForSuccess();
264 |         while (!this.cb.getState().equals(State.CLOSED)) {
265 |             try {
266 |                 ValueWrapper value = this.cache.get(TEST_KEY);
267 |                 if (this.cb.getState().equals(State.HALF_OPEN)) {
268 |                     assertNotNull("Return value is null", value);
269 |                     assertEquals("Unexpected value type",SimpleValueWrapper.class, value.getClass());
270 |                     assertEquals("Unexpected value", TEST_KEY, ((SimpleValueWrapper)value).get());
271 |                 } else {
272 |                     if (this.cb.getState().equals(State.HALF_OPEN) || this.cb.getState().equals(State.CLOSED)) {
273 |                         continue;
274 |                     }
275 |                     assertNull("Unexpected value", value);
276 |                 }
277 |             } catch (Exception e) {
278 |                 e.printStackTrace();
279 |                 fail("Unexpected exception from redis cache: " + e.getMessage());
280 |             }
281 |         }
282 |         
283 |         // circuit should be closed now
284 |         assertEquals("Unexpected circuit breaker state", CircuitBreaker.State.CLOSED, this.cb.getState());
285 |         try {
286 |             ValueWrapper value = this.cache.get(TEST_KEY);
287 |             assertNotNull("Return value is null", value);
288 |             assertEquals("Unexpected value type",SimpleValueWrapper.class, value.getClass());
289 |             assertEquals("Unexpected value", TEST_KEY, ((SimpleValueWrapper)value).get());
290 |         } catch (Exception e) {
291 |             e.printStackTrace();
292 |             fail("Unexpected exception from redis cache: " + e.getMessage());
293 |         }
294 |         try {
295 |             String val = this.cache.get(TEST_KEY, () -> TEST_KEY);
296 |             assertNotNull("Return value is null", val);
297 |             assertEquals("Unexpected value", TEST_KEY, val);
298 |         } catch (Exception e) {
299 |             e.printStackTrace();
300 |             fail("Unexpected exception from redis cache get: " + e.getMessage());
301 |         }
302 |         try {
303 |             this.cache.put(TEST_KEY, TEST_KEY);
304 |         } catch (Exception e) {
305 |             e.printStackTrace();
306 |             fail("Unexpected exception from redis cache put: " + e.getMessage());
307 |         }
308 |         try {
309 |             ValueWrapper value = this.cache.putIfAbsent(TEST_KEY, TEST_KEY);
310 |             assertNotNull("Return value is null", value);
311 |             assertEquals("Unexpected value type",SimpleValueWrapper.class, value.getClass());
312 |             assertEquals("Unexpected value", TEST_KEY, ((SimpleValueWrapper)value).get());
313 |         } catch (Exception e) {
314 |             e.printStackTrace();
315 |             fail("Unexpected exception from redis cache put: " + e.getMessage());
316 |         }
317 |         try {
318 |             this.cache.evict(TEST_KEY);
319 |         } catch (Exception e) {
320 |             e.printStackTrace();
321 |             fail("Unexpected exception from redis cache evict: " + e.getMessage());
322 |         }
323 |         try {
324 |             this.cache.clear();
325 |         } catch (Exception e) {
326 |             e.printStackTrace();
327 |             fail("Unexpected exception from redis cache clear: " + e.getMessage());
328 |         }
329 |     }
330 | 
331 |     @Test
332 |     public void testCircuitBreakerOpenedSlow() {
333 |         this.setUpForSlowFailure();
334 | 
335 |         for (int i = 0; i < CB_SLIDING_WINDOW_SIZE; i++) {
336 |             if (this.cb.getState().equals(State.CLOSED)) {
337 |                 try {
338 |                     ValueWrapper value = this.cache.get(TEST_KEY);
339 |                     assertNotNull("Return value is null", value);
340 |                     assertEquals("Unexpected value type",SimpleValueWrapper.class, value.getClass());
341 |                     assertEquals("Unexpected value", TEST_KEY, ((SimpleValueWrapper)value).get());
342 |                 } catch (Exception e) {
343 |                     e.printStackTrace();
344 |                     fail("Unexpected exception from redis cache: " + e.getMessage());
345 |                 }
346 |             }
347 |         }
348 |         
349 |         // circuit should be open now
350 |         assertEquals("Unexpected circuit breaker state", CircuitBreaker.State.OPEN, this.cb.getState());
351 |         try {
352 |             ValueWrapper value = this.cache.get(TEST_KEY);
353 |             assertEquals("Unexpected value", null, value);
354 |         } catch (Exception e) {
355 |             e.printStackTrace();
356 |             fail("Unexpected exception from redis cache: " + e.getMessage());
357 |         }
358 | 
359 |         try {
360 |             this.cache.clear();
361 |         } catch (Exception e) {
362 |             e.printStackTrace();
363 |             fail("Unexpected exception from redis cache: " + e.getMessage());
364 |         }
365 |     }
366 | 
367 | }
368 | 


--------------------------------------------------------------------------------