├── .gitignore
├── DCAwareFailoverDropwizard
    ├── .travis.yml
    ├── .gitignore
    ├── src
    │   └── main
    │   │   ├── resources
    │   │       └── META-INF
    │   │       │   └── services
    │   │       │       └── org.stuartgunter.dropwizard.cassandra.loadbalancing.LoadBalancingPolicyFactory
    │   │   └── java
    │   │       └── org
    │   │           └── adejanovski
    │   │               └── cassandra
    │   │                   └── policies
    │   │                       └── dropwizard
    │   │                           └── DCAwareFailoverRoundRobinPolicyFactory.java
    ├── README.md
    └── pom.xml
├── DCAwareFailoverRoundRobinPolicy
    ├── .travis.yml
    ├── .gitignore
    ├── src
    │   ├── main
    │   │   ├── resources
    │   │   │   └── META-INF
    │   │   │   │   └── services
    │   │   │   │       └── org.stuartgunter.dropwizard.cassandra.loadbalancing.LoadBalancingPolicyFactory
    │   │   └── java
    │   │   │   └── org
    │   │   │       └── adejanovski
    │   │   │           └── cassandra
    │   │   │               └── policies
    │   │   │                   └── DCAwareFailoverRoundRobinPolicy.java
    │   └── test
    │   │   └── java
    │   │       └── org
    │   │           └── adejanovski
    │   │               └── cassandra
    │   │                   └── policies
    │   │                       └── DCAwareFailoverRoundRobinPolicyTest.java
    ├── README.md
    └── pom.xml
├── .settings
    ├── org.eclipse.core.resources.prefs
    └── org.eclipse.m2e.core.prefs
├── .project
└── pom.xml


/.gitignore:
--------------------------------------------------------------------------------
1 | target/
2 | 


--------------------------------------------------------------------------------
/DCAwareFailoverDropwizard/.travis.yml:
--------------------------------------------------------------------------------
1 | language: java


--------------------------------------------------------------------------------
/DCAwareFailoverRoundRobinPolicy/.travis.yml:
--------------------------------------------------------------------------------
1 | language: java


--------------------------------------------------------------------------------
/.settings/org.eclipse.core.resources.prefs:
--------------------------------------------------------------------------------
1 | eclipse.preferences.version=1
2 | encoding/<project>=UTF-8
3 | 


--------------------------------------------------------------------------------
/.settings/org.eclipse.m2e.core.prefs:
--------------------------------------------------------------------------------
1 | activeProfiles=
2 | eclipse.preferences.version=1
3 | resolveWorkspaceProjects=true
4 | version=1
5 | 


--------------------------------------------------------------------------------
/DCAwareFailoverDropwizard/.gitignore:
--------------------------------------------------------------------------------
1 | /target
2 | /.settings
3 | /.classpath
4 | /.project
5 | test-output/
6 | dependency-reduced-pom.xml
7 | 


--------------------------------------------------------------------------------
/DCAwareFailoverRoundRobinPolicy/.gitignore:
--------------------------------------------------------------------------------
1 | /target
2 | /.settings
3 | /.classpath
4 | /.project
5 | test-output/
6 | dependency-reduced-pom.xml
7 | 


--------------------------------------------------------------------------------
/DCAwareFailoverDropwizard/src/main/resources/META-INF/services/org.stuartgunter.dropwizard.cassandra.loadbalancing.LoadBalancingPolicyFactory:
--------------------------------------------------------------------------------
1 | org.adejanovski.cassandra.policies.dropwizard.DCAwareFailoverRoundRobinPolicyFactory


--------------------------------------------------------------------------------
/DCAwareFailoverRoundRobinPolicy/src/main/resources/META-INF/services/org.stuartgunter.dropwizard.cassandra.loadbalancing.LoadBalancingPolicyFactory:
--------------------------------------------------------------------------------
1 | org.adejanovski.cassandra.policies.dropwizard.DCAwareFailoverRoundRobinPolicyFactory


--------------------------------------------------------------------------------
/.project:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <projectDescription>
 3 | 	<name>dcaware-failover-policy-parent</name>
 4 | 	<comment></comment>
 5 | 	<projects>
 6 | 	</projects>
 7 | 	<buildSpec>
 8 | 		<buildCommand>
 9 | 			<name>org.eclipse.m2e.core.maven2Builder</name>
10 | 			<arguments>
11 | 			</arguments>
12 | 		</buildCommand>
13 | 	</buildSpec>
14 | 	<natures>
15 | 		<nature>org.eclipse.m2e.core.maven2Nature</nature>
16 | 	</natures>
17 | </projectDescription>
18 | 


--------------------------------------------------------------------------------
/DCAwareFailoverDropwizard/src/main/java/org/adejanovski/cassandra/policies/dropwizard/DCAwareFailoverRoundRobinPolicyFactory.java:
--------------------------------------------------------------------------------
  1 | package org.adejanovski.cassandra.policies.dropwizard;
  2 | 
  3 | import org.adejanovski.cassandra.policies.DCAwareFailoverRoundRobinPolicy;
  4 | import org.stuartgunter.dropwizard.cassandra.loadbalancing.*;
  5 | 
  6 | import com.datastax.driver.core.policies.DCAwareRoundRobinPolicy;
  7 | import com.datastax.driver.core.policies.LoadBalancingPolicy;
  8 | import com.fasterxml.jackson.annotation.JsonProperty;
  9 | import com.fasterxml.jackson.annotation.JsonTypeName;
 10 | import com.datastax.driver.core.policies.LoadBalancingPolicy;
 11 | 
 12 | /**
 13 |  * A factory for configuring and building
 14 |  * {@link com.datastax.driver.core.policies.DCAwareRoundRobinPolicy} instances.
 15 |  * <b>Configuration Parameters:</b>
 16 |  * <table>
 17 |  * <tr>
 18 |  * <td>Name</td>
 19 |  * <td>Default</td>
 20 |  * <td>Description</td>
 21 |  * </tr>
 22 |  * <tr>
 23 |  * <td>localDC</td>
 24 |  * <td>No default.</td>
 25 |  * <td>The name of the local datacenter (as known by Cassandra).</td>
 26 |  * </tr>
 27 |  * <tr>
 28 |  * <td>backupDC</td>
 29 |  * <td>No default.</td>
 30 |  * <td>The name of the backup datacenter (as known by Cassandra).</td>
 31 |  * </tr>
 32 |  * <tr>
 33 |  * <td>switchBackDelayFactor</td>
 34 |  * <td>0</td>
 35 |  * <td>The connection can switch back if uptime &gt;= downtime*switchBackDelayFactor (gives time for hinted handoff to complete)</td>
 36 |  * </tr>
 37 |  * <tr>
 38 |  * <td>noSwitchBackDowntimeDelay</td>
 39 |  * <td>0</td>
 40 |  * <td>Switch back cannot happen if downtime &gt; noSwitchBackDowntimeDelay (in seconds)</td>
 41 |  * </tr>
 42 |  * </table>
 43 |  */
 44 | @JsonTypeName("dcAwareFailoverRoundRobin")
 45 | public class DCAwareFailoverRoundRobinPolicyFactory implements
 46 | 		LoadBalancingPolicyFactory {
 47 | 
 48 | 	private String localDC;
 49 | 	private String backupDC;
 50 | 	private Integer hostDownSwitchThreshold;
 51 | 	private Float switchBackDelayFactor;
 52 | 	private Integer noSwitchBackDowntimeDelay;
 53 | 
 54 | 	@JsonProperty
 55 | 	public String getLocalDC() {
 56 | 		return localDC;
 57 | 	}
 58 | 
 59 | 	@JsonProperty
 60 | 	public void setLocalDC(String localDC) {
 61 | 		this.localDC = localDC;
 62 | 	}
 63 | 
 64 | 	@JsonProperty
 65 | 	public String getBackupDC() {
 66 | 		return backupDC;
 67 | 	}
 68 | 
 69 | 	@JsonProperty
 70 | 	public void setBackupDC(String backupDC) {
 71 | 		this.backupDC = backupDC;
 72 | 	}
 73 | 
 74 | 	@JsonProperty
 75 | 	public Integer getHostDownSwitchThreshold() {
 76 | 		return hostDownSwitchThreshold;
 77 | 	}
 78 | 
 79 | 	@JsonProperty
 80 | 	public void setHostDownSwitchThreshold(Integer hostDownSwitchThreshold) {
 81 | 		this.hostDownSwitchThreshold = hostDownSwitchThreshold;
 82 | 	}
 83 | 	
 84 | 	@JsonProperty
 85 | 	public void setSwitchBackDelayFactor(Float switchBackDelayFactor) {
 86 | 		this.switchBackDelayFactor = switchBackDelayFactor;
 87 | 	}
 88 | 
 89 | 	@JsonProperty
 90 | 	public void setNoSwitchBackDowntimeDelay(Integer noSwitchBackDowntimeDelay) {
 91 | 		this.noSwitchBackDowntimeDelay = noSwitchBackDowntimeDelay;
 92 | 	}
 93 | 
 94 | 	public LoadBalancingPolicy build() {		
 95 | 		return new DCAwareFailoverRoundRobinPolicy(localDC, backupDC,
 96 | 				hostDownSwitchThreshold, switchBackDelayFactor, noSwitchBackDowntimeDelay);
 97 | 	}
 98 | 
 99 | }
100 | 


--------------------------------------------------------------------------------
/DCAwareFailoverRoundRobinPolicy/src/test/java/org/adejanovski/cassandra/policies/DCAwareFailoverRoundRobinPolicyTest.java:
--------------------------------------------------------------------------------
  1 | package org.adejanovski.cassandra.policies;
  2 | 
  3 | import java.net.InetAddress;
  4 | import java.net.UnknownHostException;
  5 | import java.util.ArrayList;
  6 | import java.util.Collection;
  7 | 
  8 | import org.adejanovski.cassandra.policies.DCAwareFailoverRoundRobinPolicy;
  9 | import org.mockito.Mockito;
 10 | import org.testng.annotations.BeforeClass;
 11 | import org.testng.annotations.BeforeMethod;
 12 | import org.testng.annotations.Test;
 13 | 
 14 | import static org.testng.Assert.*;
 15 | 
 16 | import com.datastax.driver.core.Cluster;
 17 | import com.datastax.driver.core.Host;
 18 | import com.datastax.driver.core.HostDistance;
 19 | 
 20 | public class DCAwareFailoverRoundRobinPolicyTest {
 21 | 	DCAwareFailoverRoundRobinPolicy policy;
 22 | 	Collection<Host> hosts;
 23 | 	Cluster cluster;
 24 | 
 25 | 	@BeforeMethod
 26 | 	public void setUp() throws UnknownHostException {
 27 | 		hosts = new ArrayList<Host>();
 28 | 		policy = new DCAwareFailoverRoundRobinPolicy("dc1", "dc2", 2);
 29 | 		for(int i=0;i<6;i++){
 30 | 			Host host = Mockito.mock(Host.class);
 31 | 			InetAddress address = InetAddress.getByName("127.0.0." + i);
 32 | 			Mockito.when(host.getAddress()).thenReturn(address);
 33 | 			if(i<=2){
 34 | 				Mockito.when(host.getDatacenter()).thenReturn("dc1");				
 35 | 			}else{
 36 | 				Mockito.when(host.getDatacenter()).thenReturn("dc2");
 37 | 			}
 38 | 			hosts.add(host);			
 39 | 		}		
 40 | 		
 41 | 		cluster = Mockito.mock(Cluster.class);
 42 | 		policy.init(cluster, hosts);
 43 | 		
 44 | 	}
 45 | 		
 46 | 
 47 | 	@Test
 48 | 	public void testDistance() throws UnknownHostException{
 49 | 		int i=0;
 50 | 		for(Host host:hosts){			
 51 | 			policy.onUp(host);
 52 | 			if(i<=2){
 53 | 				// dc1 is local
 54 | 				assertEquals(policy.distance(host),HostDistance.LOCAL);
 55 | 			}else{
 56 | 				assertTrue(policy.distance(host).equals(HostDistance.REMOTE) || policy.distance(host).equals(HostDistance.IGNORED));
 57 | 			}
 58 | 			i++;			
 59 | 		}
 60 | 	}
 61 | 	
 62 | 	@Test
 63 | 	public void testLostOneNode() throws UnknownHostException{
 64 | 		int i=0;		
 65 | 		for(Host host:hosts){
 66 | 			if(i==0){
 67 | 				// we lost the first node
 68 | 				// which doesn't trigger the switch
 69 | 				policy.onDown(host);
 70 | 			}
 71 | 			
 72 | 			if(i<=2){
 73 | 				// dc1 is local
 74 | 				assertEquals(policy.distance(host),HostDistance.LOCAL);
 75 | 			}else{
 76 | 				assertTrue(policy.distance(host).equals(HostDistance.REMOTE) || policy.distance(host).equals(HostDistance.IGNORED));
 77 | 			}
 78 | 			i++;			
 79 | 		}
 80 | 	}
 81 | 	
 82 | 	
 83 | 	@Test
 84 | 	public void testLostTwoNodes() throws UnknownHostException{
 85 | 		int i=0;		
 86 | 		
 87 | 		for(Host host:hosts){
 88 | 			if(i<=1){
 89 | 				// we lost the first node
 90 | 				// which doesn't trigger the switch
 91 | 				policy.onDown(host);
 92 | 			}
 93 | 			i++;
 94 | 		}
 95 | 			
 96 | 		i=0;
 97 | 		for(Host host:hosts){	
 98 | 			if(i<=2){
 99 | 				// dc1 is remote now (lost 2 nodes)
100 | 				assertTrue(policy.distance(host).equals(HostDistance.REMOTE) || policy.distance(host).equals(HostDistance.IGNORED));				
101 | 			}else{
102 | 				assertEquals(policy.distance(host),HostDistance.LOCAL);
103 | 			}
104 | 			i++;			
105 | 		}
106 | 	}
107 | 	
108 | 	@Test
109 | 	public void testSwitchBackProtection() throws UnknownHostException{
110 | 		int i=0;		
111 | 		
112 | 		for(Host host:hosts){
113 | 			if(i<=1){
114 | 				// we lost the first node
115 | 				// which doesn't trigger the switch
116 | 				policy.onDown(host);
117 | 			}
118 | 			i++;
119 | 		}
120 | 				
121 | 		for(Host host:hosts){
122 | 			// first host is back up
123 | 			policy.onUp(host);
124 | 			break;
125 | 		}
126 | 		
127 | 			
128 | 		i=0;
129 | 		for(Host host:hosts){	
130 | 			if(i<=2){
131 | 				// dc1 is still remote now (lost 2 nodes, and got 1 back)
132 | 				assertTrue(policy.distance(host).equals(HostDistance.REMOTE) || policy.distance(host).equals(HostDistance.IGNORED));				
133 | 			}else{
134 | 				assertEquals(policy.distance(host),HostDistance.LOCAL);
135 | 			}
136 | 			i++;			
137 | 		}
138 | 	}
139 | }
140 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  2 | 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  3 | 	<modelVersion>4.0.0</modelVersion>
  4 | 
  5 | 	<groupId>com.github.adejanovski.cassandra.policies</groupId>
  6 | 	<artifactId>dcaware-failover-policy-parent</artifactId>
  7 | 	<version>1.1.0</version>
  8 | 	<packaging>pom</packaging>
  9 | 	<name>DCAwareFailoverRoundRobinPolicy</name>	
 10 | 	<description>DC Aware Failover Round Robin Policy for Datastax Java Driver</description>	
 11 | 	<url>https://github.com/adejanovski/cassandra-dcaware-failover</url>
 12 | 
 13 | 	<inceptionYear>2015</inceptionYear>
 14 | 	<licenses>
 15 | 		<license>
 16 | 			<name>The Apache Software License, Version 2.0</name>
 17 | 			<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
 18 | 			<distribution>repo</distribution>
 19 | 		</license>
 20 | 	</licenses>
 21 | 
 22 | 	<developers>
 23 | 		<developer>
 24 | 			<id>adejanovski</id>
 25 | 			<name>Alexander Dejanovski</name>
 26 | 			<roles>
 27 | 				<role>developer</role>
 28 | 			</roles>
 29 | 		</developer>
 30 | 	</developers>
 31 | 
 32 | 	<modules>
 33 | 		<module>DCAwareFailoverRoundRobinPolicy</module>
 34 | 		<module>DCAwareFailoverDropwizard</module>
 35 | 	</modules>
 36 | 
 37 | 	<properties>
 38 | 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 39 | 		<java.version>1.7</java.version>
 40 | 	</properties>
 41 | 
 42 | 	<build>
 43 | 		<plugins>
 44 | 			<plugin>
 45 | 				<artifactId>maven-compiler-plugin</artifactId>
 46 | 				<version>3.1</version>
 47 | 				<configuration>
 48 | 					<source>${java.version}</source>
 49 | 					<target>${java.version}</target>
 50 | 					<optimize>true</optimize>
 51 | 					<showDeprecation>true</showDeprecation>
 52 | 					<showWarnings>true</showWarnings>
 53 | 				</configuration>
 54 | 			</plugin>
 55 | 		</plugins>
 56 | 
 57 | 	</build>
 58 | 
 59 | 	<profiles>
 60 | 		<profile>
 61 | 			<id>release</id>
 62 | 			<properties>
 63 | 				<env>default</env>
 64 | 			</properties>
 65 | 			<build>
 66 | 				<plugins>
 67 | 					<plugin>
 68 | 						<artifactId>maven-surefire-plugin</artifactId>
 69 | 						<version>2.17</version>
 70 | 						<configuration>
 71 | 							<skip>true</skip>
 72 | 						</configuration>
 73 | 						<executions>
 74 | 							<execution>
 75 | 								<phase>test</phase>
 76 | 								<goals>
 77 | 									<goal>test</goal>
 78 | 								</goals>
 79 | 								<configuration>
 80 | 									<skip>true</skip>
 81 | 								</configuration>
 82 | 							</execution>
 83 | 						</executions>
 84 | 					</plugin>
 85 | 					<plugin>
 86 | 						<artifactId>maven-compiler-plugin</artifactId>
 87 | 						<configuration>
 88 | 							<compilerArgument>-Xlint:all</compilerArgument>
 89 | 							<showWarnings>true</showWarnings>
 90 | 							<showDeprecation>true</showDeprecation>
 91 | 						</configuration>
 92 | 					</plugin>
 93 | 					<plugin>
 94 | 						<groupId>org.apache.maven.plugins</groupId>
 95 | 						<artifactId>maven-javadoc-plugin</artifactId>
 96 | 						<version>2.10.3</version>
 97 | 						<configuration>
 98 | 							<additionalparam>-Xdoclint:none</additionalparam>
 99 | 						</configuration>
100 | 					</plugin>
101 | 					<plugin>
102 | 						<groupId>org.apache.maven.plugins</groupId>
103 | 						<artifactId>maven-source-plugin</artifactId>
104 | 						<version>2.4</version>
105 | 						<executions>
106 | 							<execution>
107 | 								<id>attach-sources</id>
108 | 								<phase>verify</phase>
109 | 								<goals>
110 | 									<goal>jar-no-fork</goal>
111 | 								</goals>
112 | 							</execution>
113 | 						</executions>
114 | 					</plugin>
115 | 					<plugin>
116 | 						<groupId>org.apache.maven.plugins</groupId>
117 | 						<artifactId>maven-gpg-plugin</artifactId>
118 | 						<executions>
119 | 							<execution>
120 | 								<id>sign-artifacts</id>
121 | 								<phase>verify</phase>
122 | 								<goals>
123 | 									<goal>sign</goal>
124 | 								</goals>
125 | 							</execution>
126 | 						</executions>
127 | 					</plugin>
128 | 
129 | 
130 | 				</plugins>
131 | 			</build>
132 | 		</profile>
133 | 	</profiles>
134 | 
135 | 	<scm>
136 | 		<connection>scm:git:git@github.com:adejanovski/cassandra-dcaware-failover.git</connection>
137 | 		<developerConnection>scm:git:git@github.com:adejanovski/cassandra-dcaware-failover.git</developerConnection>
138 | 		<url>https://github.com/com:adejanovski/cassandra-dcaware-failover</url>
139 | 		<tag>HEAD</tag>
140 | 	</scm>
141 | 
142 | </project>
143 | 


--------------------------------------------------------------------------------
/DCAwareFailoverRoundRobinPolicy/README.md:
--------------------------------------------------------------------------------
 1 | # DC Aware Failover Round Robin Policy for Datastax Java Driver
 2 | 
 3 | Properly handling failover in multi DC Cassandra clusters with LOCAL_* CL usually require writing specific code to properly process failover scenarios.
 4 | 
 5 | The Cassandra setups that fail delivering automatic DC failure handling are those with 2 DCs or those with 1+ real time DCs and 1+ analytical/search DC.
 6 | 
 7 | Having a 2 DC cluster and using CL.QUORUM will lead to systematic failure when a DC goes down. 
 8 | Using CL.ONE will lead to inconsistencies if a DC goes down and comes back up after a while.
 9 | Using CL.LOCAL_QUORUM with DCAwareRoundRobinPolicy will segregate the traffic to a single DC, but the load balancing policy will prevent switching to the other DC in case of failure (in its basic configuration).
10 | This is a conservative behavior, because if done differently, when the primary DC comes back up it will probably not be up to date, and it could even be out of the hinted handoff window (which will require a full repair).
11 | In case you have an analytical or a search DC, you must segregate traffic in order to guarantee low latencies, which forces the use of CL.LOCAL_* consistencies. The behavior in case of failure is the same as the one above.
12 | 
13 | Handling those failure scenarios has to be implemented in client applications, by doing the following :
14 | * detect how many nodes are down in the local DC
15 | * if too many nodes are down to allow queries to succeed (depends on the replication factor), the app has to switch to a backup dc (either having two connections and switching between them, either recreating a new connection with a different LB policy conf).
16 | * Switching automatically back at any time to the local DC must be prevented because repair might be necessary in order to regain consistency
17 |  
18 |  
19 | The DCAwareFailoverRoundRobinPolicy is based on the DCAwareRoundRobinPolicy and handles the above one way switch without having to code it. It takes the following parameters : 
20 | * localDc : the primary DC of the app
21 | * backupDc : the DC to switch to in case the primary DC loses a defined number of nodes
22 | * hostDownSwitchThreshold : the number of lost nodes that will trigger the switch
23 | * switchBackDelayFactor : The connection can switch back if uptime >= downtime*switchBackDelayFactor (gives time for hinted handoff to complete)
24 | * noSwitchBackDowntimeDelay : Switch back cannot happen if downtime > noSwitchBackDowntimeDelay (in seconds) - usually the hinted handoff window (default in Cassandra is 3h)
25 |   
26 | This policy does a one way switch to the backup DC to prevent inconsistencies, and the connection must be recreated in order to switch back to the local DC (which should require either rebooting the app, or recreating the Cluster and Session objects), unless auto back switch conditions are fulfilled : 
27 | * Local DC has regained enough nodes
28 | * uptime is superior to downtime*switchBackDelayFactor
29 | * downtime has been lower than noSwitchBackDowntimeDelay
30 | 
31 | 
32 | # Usage
33 | 
34 | Include this project in your classpath and use it as any other load balancing policy when creating the cluster object:
35 | 
36 | ```java
37 | Cluster cluster = Cluster.builder()
38 | 			         .addContactPoints("127.0.0.1","127.0.0.2","127.0.0.3","127.0.0.4","127.0.0.5")
39 | 			         .withLoadBalancingPolicy(new org.adejanovski.cassandra.policies.DCAwareRoundRobinPolicy("primaryDc","backupDc",2))			        
40 | 			         .build();
41 | ```			         
42 | 
43 | The above will segregate coordination traffic to DC 'primaryDc' and if at least 2 nodes are lost in this DC, switch to 'backupDc' and never go back.
44 | 
45 | To authorize auto back switch you can add the switchBackDelayFactor and noSwitchBackDowntimeDelay : 
46 | 
47 | ```java
48 | Cluster cluster = Cluster.builder()
49 | 			         .addContactPoints("127.0.0.1","127.0.0.2","127.0.0.3","127.0.0.4","127.0.0.5")
50 | 			         .withLoadBalancingPolicy(new org.adejanovski.cassandra.policies.DCAwareRoundRobinPolicy("primaryDc","backupDc", 2, 1.5, 10800))			        
51 | 			         .build();
52 | ``` 
53 | 
54 | The above will allow automatically switching back if conditions are fulfilled : 
55 | * downtime didn't last more than 3 hours (10800 seconds)
56 | * uptime since local DC recovery has lasted more than downtime*1.5 (if downtime was 30 mn, back switch will be prevented for 45 mn after local DC recovery) 
57 | 
58 | # Usage in Dropwizard
59 | 
60 | The DCAwareFailoverRoundRobinPolicy is "dropwizard-ready".
61 |  
62 | Use it by declaring it in the yml config file, and it can be combined with any appropriate parent policy:
63 | 
64 | ```xml
65 | loadBalancingPolicy:
66 |     type: tokenAware
67 |     shuffleReplicas: true
68 |     subPolicy:
69 |       type: dcAwareFailoverRoundRobin
70 |       localDC: primaryDc
71 |       backupDC: backupDc
72 |       hostDownSwitchThreshold: 2
73 |       switchBackDelayFactor: 1.5
74 |       noSwitchBackDowntimeDelay: 10800 
75 | ```
76 | 
77 | 			         
78 | 			         


--------------------------------------------------------------------------------
/DCAwareFailoverDropwizard/README.md:
--------------------------------------------------------------------------------
 1 | # DC Aware Failover Round Robin Policy for Datastax Java Driver
 2 | 
 3 | Properly handling failover in multi DC Cassandra clusters with LOCAL_* CL usually require writing specific code to properly process failover scenarios.
 4 | 
 5 | The Cassandra setups that fail delivering automatic DC failure handling are those with 2 DCs or those with 1+ real time DCs and 1+ analytical/search DC.
 6 | 
 7 | Having a 2 DC cluster and using CL.QUORUM will lead to systematic failure when a DC goes down. 
 8 | Using CL.ONE will lead to inconsistencies if a DC goes down and comes back up after a while.
 9 | Using CL.LOCAL_QUORUM with DCAwareRoundRobinPolicy will segregate the traffic to a single DC, but the load balancing policy will prevent switching to the other DC in case of failure (in its basic configuration).
10 | This is a conservative behavior, because if done differently, when the primary DC comes back up it will probably not be up to date, and it could even be out of the hinted handoff window (which will require a full repair).
11 | In case you have an analytical or a search DC, you must segregate traffic in order to guarantee low latencies, which forces the use of CL.LOCAL_* consistencies. The behavior in case of failure is the same as the one above.
12 | 
13 | Handling those failure scenarios has to be implemented in client applications, by doing the following :
14 | * detect how many nodes are down in the local DC
15 | * if too many nodes are down to allow queries to succeed (depends on the replication factor), the app has to switch to a backup dc (either having two connections and switching between them, either recreating a new connection with a different LB policy conf).
16 | * Switching automatically back at any time to the local DC must be prevented because repair might be necessary in order to regain consistency
17 |  
18 |  
19 | The DCAwareFailoverRoundRobinPolicy is based on the DCAwareRoundRobinPolicy and handles the above one way switch without having to code it. It takes the following parameters : 
20 | * localDc : the primary DC of the app
21 | * backupDc : the DC to switch to in case the primary DC loses a defined number of nodes
22 | * hostDownSwitchThreshold : the number of lost nodes that will trigger the switch
23 | * switchBackDelayFactor : The connection can switch back if uptime >= downtime*switchBackDelayFactor (gives time for hinted handoff to complete)
24 | * noSwitchBackDowntimeDelay : Switch back cannot happen if downtime > noSwitchBackDowntimeDelay (in seconds) - usually the hinted handoff window (default in Cassandra is 3h)
25 |   
26 | This policy does a one way switch to the backup DC to prevent inconsistencies, and the connection must be recreated in order to switch back to the local DC (which should require either rebooting the app, or recreating the Cluster and Session objects), unless auto back switch conditions are fulfilled : 
27 | * Local DC has regained enough nodes
28 | * uptime is superior to downtime*switchBackDelayFactor
29 | * downtime has been lower than noSwitchBackDowntimeDelay
30 | 
31 | 
32 | # Usage
33 | 
34 | Include this project in your classpath and use it as any other load balancing policy when creating the cluster object:
35 | 
36 | ```java
37 | Cluster cluster = Cluster.builder()
38 | 			         .addContactPoints("127.0.0.1","127.0.0.2","127.0.0.3","127.0.0.4","127.0.0.5")
39 | 			         .withLoadBalancingPolicy(new org.adejanovski.cassandra.policies.DCAwareRoundRobinPolicy("primaryDc","backupDc",2))			        
40 | 			         .build();
41 | ```			         
42 | 
43 | The above will segregate coordination traffic to DC 'primaryDc' and if at least 2 nodes are lost in this DC, switch to 'backupDc' and never go back.
44 | 
45 | To authorize auto back switch you can add the switchBackDelayFactor and noSwitchBackDowntimeDelay : 
46 | 
47 | ```java
48 | Cluster cluster = Cluster.builder()
49 | 			         .addContactPoints("127.0.0.1","127.0.0.2","127.0.0.3","127.0.0.4","127.0.0.5")
50 | 			         .withLoadBalancingPolicy(new org.adejanovski.cassandra.policies.DCAwareRoundRobinPolicy("primaryDc","backupDc", 2, 1.5, 10800))			        
51 | 			         .build();
52 | ``` 
53 | 
54 | The above will allow automatically switching back if conditions are fulfilled : 
55 | * downtime didn't last more than 3 hours (10800 seconds)
56 | * uptime since local DC recovery has lasted more than downtime*1.5 (if downtime was 30 mn, back switch will be prevented for 45 mn after local DC recovery) 
57 | 
58 | # Usage in Dropwizard
59 | 
60 | The DCAwareFailoverRoundRobinPolicy is "dropwizard-ready".
61 |  
62 | Use it by declaring it in the yml config file, and it can be combined with any appropriate parent policy:
63 | 
64 | ```xml
65 | loadBalancingPolicy:
66 |     type: tokenAware
67 |     shuffleReplicas: true
68 |     subPolicy:
69 |       type: dcAwareFailoverRoundRobin
70 |       localDC: primaryDc
71 |       backupDC: backupDc
72 |       hostDownSwitchThreshold: 2
73 |       switchBackDelayFactor: 1.5
74 |       noSwitchBackDowntimeDelay: 10800 
75 | ```
76 | 
77 | ### Build status
78 | 
79 | [![Build Status](https://travis-ci.org/adejanovski/cassandra-dcaware-failover.svg)](https://travis-ci.org/adejanovski/cassandra-dcaware-failover)
80 | 
81 | 			         
82 | 			         
83 | 


--------------------------------------------------------------------------------
/DCAwareFailoverRoundRobinPolicy/pom.xml:
--------------------------------------------------------------------------------
  1 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  2 | 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  3 | 	<modelVersion>4.0.0</modelVersion>
  4 | 
  5 | 	<parent>
  6 | 		<groupId>com.github.adejanovski.cassandra.policies</groupId>
  7 | 		<artifactId>dcaware-failover-policy-parent</artifactId> 
  8 | 		<version>1.1.0</version>
  9 | 	</parent>
 10 | 	
 11 | 	<artifactId>dcaware-failover-policy</artifactId>
 12 | 	<packaging>jar</packaging>
 13 | 
 14 | 	<name>DCAwareFailoverRoundRobinPolicy</name>
 15 | 	<url>http://maven.apache.org</url>
 16 | 
 17 | 	<properties>
 18 | 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 19 | 		<cassandra.driver.version>2.1.9</cassandra.driver.version>
 20 | 	</properties>
 21 | 
 22 | 	<dependencies>
 23 | 		
 24 | 		<dependency>
 25 | 			<groupId>com.datastax.cassandra</groupId>
 26 | 			<artifactId>cassandra-driver-core</artifactId>
 27 | 			<version>${cassandra.driver.version}</version>			
 28 | 		</dependency>
 29 | 	
 30 | 		<dependency>
 31 | 			<groupId>org.testng</groupId>
 32 | 			<artifactId>testng</artifactId>
 33 | 			<version>6.9.4</version>
 34 | 			<scope>test</scope>
 35 | 		</dependency>
 36 | 		
 37 | 		<dependency>
 38 | 			<groupId>org.mockito</groupId>
 39 | 			<artifactId>mockito-all</artifactId>
 40 | 			<version>1.10.8</version>
 41 | 			<scope>test</scope>
 42 | 		</dependency>
 43 | 
 44 | 	</dependencies>
 45 | 
 46 | 	<build>
 47 | 		<resources>
 48 | 			<resource>
 49 | 				<directory>META-INF/services</directory>
 50 | 			</resource>
 51 | 		</resources>
 52 | 		
 53 | 		<pluginManagement>
 54 | 			<plugins>
 55 | 				<plugin>
 56 | 					<groupId>org.apache.maven.plugins</groupId>
 57 | 					<artifactId>maven-assembly-plugin</artifactId>
 58 | 					<version>2.4</version>
 59 | 					<dependencies>
 60 | 						<dependency>
 61 | 							<groupId>org.apache.apache.resources</groupId>
 62 | 							<artifactId>apache-source-release-assembly-descriptor</artifactId>
 63 | 							<version>1.0.3</version>
 64 | 						</dependency>
 65 | 					</dependencies>
 66 | 				</plugin>
 67 | 				<plugin>
 68 | 					<groupId>org.apache.maven.plugins</groupId>
 69 | 					<artifactId>maven-clean-plugin</artifactId>
 70 | 					<version>2.5</version>
 71 | 				</plugin>
 72 | 				<plugin>
 73 | 					<groupId>org.apache.maven.plugins</groupId>
 74 | 					<artifactId>maven-compiler-plugin</artifactId>
 75 | 					<version>3.0</version>
 76 | 				</plugin>
 77 | 				<plugin>
 78 | 					<groupId>org.apache.maven.plugins</groupId>
 79 | 					<artifactId>maven-deploy-plugin</artifactId>
 80 | 					<version>2.7</version>
 81 | 				</plugin>
 82 | 				<plugin>
 83 | 					<groupId>org.apache.maven.plugins</groupId>
 84 | 					<artifactId>maven-failsafe-plugin</artifactId>
 85 | 					<version>2.14</version>
 86 | 				</plugin>
 87 | 				<plugin>
 88 | 					<groupId>org.apache.maven.plugins</groupId>
 89 | 					<artifactId>maven-gpg-plugin</artifactId>
 90 | 					<version>1.4</version>
 91 | 					<configuration>
 92 | 						<passphrase>${gpg.passphrase}</passphrase>
 93 | 						<useAgent>${gpg.useagent}</useAgent>
 94 | 					</configuration>
 95 | 				</plugin>
 96 | 				<plugin>
 97 | 					<groupId>org.apache.maven.plugins</groupId>
 98 | 					<artifactId>maven-install-plugin</artifactId>
 99 | 					<version>2.4</version>
100 | 				</plugin>
101 | 				<plugin>
102 | 					<groupId>org.apache.maven.plugins</groupId>
103 | 					<artifactId>maven-jar-plugin</artifactId>
104 | 					<version>2.4</version>
105 | 				</plugin>
106 | 				<plugin>
107 | 					<groupId>org.apache.maven.plugins</groupId>
108 | 					<artifactId>maven-javadoc-plugin</artifactId>
109 | 					<version>2.9</version>
110 | 				</plugin>
111 | 				<plugin>
112 | 					<groupId>org.apache.maven.plugins</groupId>
113 | 					<artifactId>maven-release-plugin</artifactId>
114 | 					<version>2.4</version>
115 | 					<configuration>
116 | 						<goals>deploy</goals>
117 | 					</configuration>
118 | 				</plugin>
119 | 				<plugin>
120 | 					<groupId>org.apache.maven.plugins</groupId>
121 | 					<artifactId>maven-resources-plugin</artifactId>
122 | 					<version>2.6</version>
123 | 				</plugin>
124 | 				<plugin>
125 | 					<groupId>org.apache.maven.plugins</groupId>
126 | 					<artifactId>maven-site-plugin</artifactId>
127 | 					<version>3.2</version>
128 | 				</plugin>
129 | 				<plugin>
130 | 					<groupId>org.apache.maven.plugins</groupId>
131 | 					<artifactId>maven-surefire-plugin</artifactId>
132 | 					<version>2.14</version>
133 | 				</plugin>
134 | 				<plugin>
135 | 					<groupId>org.apache.maven.plugins</groupId>
136 | 					<artifactId>maven-source-plugin</artifactId>
137 | 					<version>2.2.1</version>
138 | 				</plugin>
139 | 				<plugin>
140 | 					<groupId>org.codehaus.mojo</groupId>
141 | 					<artifactId>animal-sniffer-maven-plugin</artifactId>
142 | 					<version>1.9</version>
143 | 				</plugin>
144 | 				<plugin>
145 | 					<groupId>org.codehaus.mojo</groupId>
146 | 					<artifactId>build-helper-maven-plugin</artifactId>
147 | 					<version>1.7</version>
148 | 				</plugin>
149 | 				<plugin>
150 | 					<groupId>org.codehaus.mojo</groupId>
151 | 					<artifactId>cassandra-maven-plugin</artifactId>
152 | 					<version>1.2.1-1</version>
153 | 				</plugin>
154 | 
155 | 				<!--This plugin's configuration is used to store Eclipse m2e settings 
156 | 					only. It has no influence on the Maven build itself. -->
157 | 				<plugin>
158 | 					<groupId>org.eclipse.m2e</groupId>
159 | 					<artifactId>lifecycle-mapping</artifactId>
160 | 					<version>1.0.0</version>
161 | 					<configuration>
162 | 						<lifecycleMappingMetadata>
163 | 							<pluginExecutions>
164 | 								<pluginExecution>
165 | 									<pluginExecutionFilter>
166 | 										<groupId>
167 | 											org.codehaus.mojo
168 | 										</groupId>
169 | 										<artifactId>
170 | 											build-helper-maven-plugin
171 | 										</artifactId>
172 | 										<versionRange>
173 | 											[1.7,)
174 | 										</versionRange>
175 | 										<goals>
176 | 											<goal>
177 | 												reserve-network-port
178 | 											</goal>
179 | 										</goals>
180 | 									</pluginExecutionFilter>
181 | 									<action>
182 | 										<ignore></ignore>
183 | 									</action>
184 | 								</pluginExecution>
185 | 							</pluginExecutions>
186 | 						</lifecycleMappingMetadata>
187 | 					</configuration>
188 | 				</plugin>
189 | 			</plugins>
190 | 		</pluginManagement>
191 | 		<plugins>			
192 | 			<plugin>
193 | 				<artifactId>maven-compiler-plugin</artifactId>
194 | 				<configuration>
195 | 					<source>1.6</source>
196 | 					<target>1.6</target>
197 | 				</configuration>
198 | 			</plugin>
199 | 			<plugin>
200 | 				<artifactId>maven-surefire-plugin</artifactId>
201 | 				<configuration>
202 | 					<includes>
203 | 						<include>**/*Test.class</include>
204 | 					</includes>					
205 | 				</configuration>
206 | 			</plugin>
207 | 			<plugin>
208 | 				<artifactId>maven-failsafe-plugin</artifactId>
209 | 				<configuration>
210 | 					<includes>
211 | 						<include>**/*Test.class</include>
212 | 					</includes>					
213 | 				</configuration>
214 | 				<executions>
215 | 					<execution>
216 | 						<goals>
217 | 							<goal>integration-test</goal>
218 | 							<goal>verify</goal>
219 | 						</goals>
220 | 					</execution>
221 | 				</executions>
222 | 			</plugin>
223 | 		</plugins>
224 | 	</build>
225 | 
226 | 	<profiles>
227 | 		<profile>
228 | 			<id>default</id>
229 | 			<properties>
230 | 				<env>default</env>
231 | 			</properties>
232 | 			<activation>
233 | 				<activeByDefault>true</activeByDefault>
234 | 			</activation>
235 | 			<build>
236 | 				<plugins>
237 | 					<plugin>
238 | 						<artifactId>maven-surefire-plugin</artifactId>
239 | 						<version>2.17</version>
240 | 						<configuration>
241 | 							<skip>true</skip>
242 | 						</configuration>
243 | 						<executions>
244 | 							<execution>
245 | 								<phase>test</phase>
246 | 								<goals>
247 | 									<goal>test</goal>
248 | 								</goals>
249 | 								<configuration>
250 | 									<skip>true</skip>
251 | 								</configuration>
252 | 							</execution>
253 | 						</executions>
254 | 					</plugin>
255 | 					<plugin>
256 | 						<artifactId>maven-compiler-plugin</artifactId>
257 | 						<configuration>
258 | 							<compilerArgument>-Xlint:all</compilerArgument>
259 | 							<showWarnings>true</showWarnings>
260 | 							<showDeprecation>true</showDeprecation>
261 | 						</configuration>
262 | 					</plugin>
263 | 					<plugin>
264 | 						<groupId>org.apache.maven.plugins</groupId>
265 | 						<artifactId>maven-javadoc-plugin</artifactId>
266 | 						<configuration>
267 | 					      <additionalparam>-Xdoclint:none</additionalparam>
268 | 					    </configuration>
269 | 						<version>2.10.3</version>
270 | 					</plugin>					
271 | 
272 | 				</plugins>
273 | 			</build>
274 | 		</profile>
275 | 
276 | 		<profile>
277 | 			<id>release</id>
278 | 			<properties>
279 | 				<env>default</env>
280 | 			</properties>
281 | 			<build>
282 | 				<plugins>
283 | 					<plugin>
284 | 						<artifactId>maven-surefire-plugin</artifactId>
285 | 						<version>2.17</version>
286 | 						<configuration>
287 | 							<skip>true</skip>
288 | 						</configuration>
289 | 						<executions>
290 | 							<execution>
291 | 								<phase>test</phase>
292 | 								<goals>
293 | 									<goal>test</goal>
294 | 								</goals>
295 | 								<configuration>
296 | 									<skip>true</skip>
297 | 								</configuration>
298 | 							</execution>
299 | 						</executions>
300 | 					</plugin>
301 | 					<plugin>
302 | 						<artifactId>maven-compiler-plugin</artifactId>
303 | 						<configuration>
304 | 							<compilerArgument>-Xlint:all</compilerArgument>
305 | 							<showWarnings>true</showWarnings>
306 | 							<showDeprecation>true</showDeprecation>
307 | 						</configuration>
308 | 					</plugin>
309 | 					<plugin>
310 | 						<groupId>org.apache.maven.plugins</groupId>
311 | 						<artifactId>maven-javadoc-plugin</artifactId>
312 | 						<version>2.10.3</version>
313 | 						<configuration>
314 | 					      <additionalparam>-Xdoclint:none</additionalparam>
315 | 					    </configuration>
316 | 					</plugin>
317 | 					<plugin>
318 | 						<groupId>org.apache.maven.plugins</groupId>
319 | 						<artifactId>maven-source-plugin</artifactId>
320 | 						<version>2.4</version>
321 | 						<executions>
322 | 							<execution>
323 | 								<id>attach-sources</id>
324 | 								<phase>verify</phase>
325 | 								<goals>
326 | 									<goal>jar-no-fork</goal>
327 | 								</goals>
328 | 							</execution>
329 | 						</executions>
330 | 					</plugin>
331 | 					<plugin>
332 | 						<groupId>org.apache.maven.plugins</groupId>
333 | 						<artifactId>maven-gpg-plugin</artifactId>
334 | 						<executions>
335 | 							<execution>
336 | 								<id>sign-artifacts</id>
337 | 								<phase>verify</phase>
338 | 								<goals>
339 | 									<goal>sign</goal>
340 | 								</goals>
341 | 							</execution>
342 | 						</executions>
343 | 					</plugin>
344 | 
345 | 
346 | 				</plugins>
347 | 			</build>
348 | 		</profile>
349 | 
350 | 		<profile>
351 | 			<id>test</id>
352 | 			<properties>
353 | 				<env>default</env>
354 | 			</properties>
355 | 			<build>
356 | 				<plugins>
357 | 					<plugin>
358 | 						<groupId>org.apache.maven.plugins</groupId>
359 | 						<artifactId>maven-surefire-plugin</artifactId>
360 | 						<version>2.18.1</version>						
361 | 					</plugin>
362 | 				</plugins>
363 | 			</build>
364 | 		</profile>
365 | 	</profiles>
366 | 
367 | 	<scm>
368 | 		<connection>scm:git:git@github.com:adejanovski/cassandra-dcaware-failover.git</connection>
369 | 		<developerConnection>scm:git:git@github.com:adejanovski/cassandra-dcaware-failover.git</developerConnection>
370 | 		<url>https://github.com/com:adejanovski/cassandra-dcaware-failover</url>
371 | 		<tag>HEAD</tag>
372 | 	</scm>
373 | 		
374 | </project>
375 | 


--------------------------------------------------------------------------------
/DCAwareFailoverDropwizard/pom.xml:
--------------------------------------------------------------------------------
  1 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  2 | 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  3 | 	<modelVersion>4.0.0</modelVersion>
  4 | 
  5 | 	<parent>
  6 | 		<groupId>com.github.adejanovski.cassandra.policies</groupId>
  7 | 		<artifactId>dcaware-failover-policy-parent</artifactId> 
  8 | 		<version>1.1.0</version>
  9 | 	</parent>
 10 | 
 11 | 	<artifactId>dcaware-failover-policy-dropwizard</artifactId>
 12 | 	<packaging>jar</packaging>
 13 | 
 14 | 	<name>DCAwareFailoverDropwizard</name>
 15 | 	<url>https://github.com/adejanovski/cassandra-dcaware-failover</url>
 16 | 
 17 | 	<properties>
 18 | 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 19 | 		<cassandra.driver.version>2.1.9</cassandra.driver.version>
 20 | 		<scassandra.version>0.10.0</scassandra.version>	
 21 | 	</properties>
 22 | 
 23 | 	<dependencies>
 24 | 		
 25 | 		<dependency>
 26 | 			<groupId>com.github.adejanovski.cassandra.policies</groupId>
 27 | 			<artifactId>dcaware-failover-policy</artifactId> 
 28 | 			<version>${project.parent.version}</version>
 29 | 			<exclusions>				
 30 | 				<exclusion>
 31 | 					<artifactId>metrics-core</artifactId>
 32 | 					<groupId>com.codahale.metrics</groupId>
 33 | 				</exclusion>
 34 | 			</exclusions>
 35 | 		</dependency>
 36 | 
 37 | 		<dependency>
 38 | 			<groupId>org.stuartgunter</groupId>
 39 | 			<artifactId>dropwizard-cassandra</artifactId>
 40 | 			<version>1.1-dw0.8-cs2.1</version>
 41 | 			<exclusions>
 42 | 				<exclusion>
 43 | 					<groupId>com.datastax.cassandra</groupId>
 44 | 					<artifactId>cassandra-driver-core</artifactId>
 45 | 				</exclusion>
 46 | 			</exclusions>
 47 | 		</dependency>
 48 | 
 49 | 		<dependency>
 50 | 			<groupId>org.testng</groupId>
 51 | 			<artifactId>testng</artifactId>
 52 | 			<version>6.9.4</version>
 53 | 			<scope>test</scope>
 54 | 		</dependency>
 55 | 
 56 | 
 57 | 	</dependencies>
 58 | 
 59 | 	<build>
 60 | 		<resources>
 61 | 			<resource>
 62 | 				<directory>META-INF/services</directory>
 63 | 			</resource>
 64 | 		</resources>
 65 | 		
 66 | 		<pluginManagement>
 67 | 			<plugins>
 68 | 				<plugin>
 69 | 					<groupId>org.apache.maven.plugins</groupId>
 70 | 					<artifactId>maven-assembly-plugin</artifactId>
 71 | 					<version>2.4</version>
 72 | 					<dependencies>
 73 | 						<dependency>
 74 | 							<groupId>org.apache.apache.resources</groupId>
 75 | 							<artifactId>apache-source-release-assembly-descriptor</artifactId>
 76 | 							<version>1.0.3</version>
 77 | 						</dependency>
 78 | 					</dependencies>
 79 | 				</plugin>
 80 | 				<plugin>
 81 | 					<groupId>org.apache.maven.plugins</groupId>
 82 | 					<artifactId>maven-clean-plugin</artifactId>
 83 | 					<version>2.5</version>
 84 | 				</plugin>
 85 | 				<plugin>
 86 | 					<groupId>org.apache.maven.plugins</groupId>
 87 | 					<artifactId>maven-compiler-plugin</artifactId>
 88 | 					<version>3.0</version>
 89 | 				</plugin>
 90 | 				<plugin>
 91 | 					<groupId>org.apache.maven.plugins</groupId>
 92 | 					<artifactId>maven-deploy-plugin</artifactId>
 93 | 					<version>2.7</version>
 94 | 				</plugin>
 95 | 				<plugin>
 96 | 					<groupId>org.apache.maven.plugins</groupId>
 97 | 					<artifactId>maven-failsafe-plugin</artifactId>
 98 | 					<version>2.14</version>
 99 | 				</plugin>
100 | 				<plugin>
101 | 					<groupId>org.apache.maven.plugins</groupId>
102 | 					<artifactId>maven-gpg-plugin</artifactId>
103 | 					<version>1.4</version>
104 | 					<configuration>
105 | 						<passphrase>${gpg.passphrase}</passphrase>
106 | 						<useAgent>${gpg.useagent}</useAgent>
107 | 					</configuration>
108 | 				</plugin>
109 | 				<plugin>
110 | 					<groupId>org.apache.maven.plugins</groupId>
111 | 					<artifactId>maven-install-plugin</artifactId>
112 | 					<version>2.4</version>
113 | 				</plugin>
114 | 				<plugin>
115 | 					<groupId>org.apache.maven.plugins</groupId>
116 | 					<artifactId>maven-jar-plugin</artifactId>
117 | 					<version>2.4</version>
118 | 				</plugin>
119 | 				<plugin>
120 | 					<groupId>org.apache.maven.plugins</groupId>
121 | 					<artifactId>maven-javadoc-plugin</artifactId>
122 | 					<version>2.9</version>
123 | 				</plugin>
124 | 				<plugin>
125 | 					<groupId>org.apache.maven.plugins</groupId>
126 | 					<artifactId>maven-release-plugin</artifactId>
127 | 					<version>2.4</version>
128 | 					<configuration>
129 | 						<goals>deploy</goals>
130 | 					</configuration>
131 | 				</plugin>
132 | 				<plugin>
133 | 					<groupId>org.apache.maven.plugins</groupId>
134 | 					<artifactId>maven-resources-plugin</artifactId>
135 | 					<version>2.6</version>
136 | 				</plugin>
137 | 				<plugin>
138 | 					<groupId>org.apache.maven.plugins</groupId>
139 | 					<artifactId>maven-site-plugin</artifactId>
140 | 					<version>3.2</version>
141 | 				</plugin>
142 | 				<plugin>
143 | 					<groupId>org.apache.maven.plugins</groupId>
144 | 					<artifactId>maven-surefire-plugin</artifactId>
145 | 					<version>2.14</version>
146 | 				</plugin>
147 | 				<plugin>
148 | 					<groupId>org.apache.maven.plugins</groupId>
149 | 					<artifactId>maven-source-plugin</artifactId>
150 | 					<version>2.2.1</version>
151 | 				</plugin>
152 | 				<plugin>
153 | 					<groupId>org.codehaus.mojo</groupId>
154 | 					<artifactId>animal-sniffer-maven-plugin</artifactId>
155 | 					<version>1.9</version>
156 | 				</plugin>
157 | 				<plugin>
158 | 					<groupId>org.codehaus.mojo</groupId>
159 | 					<artifactId>build-helper-maven-plugin</artifactId>
160 | 					<version>1.7</version>
161 | 				</plugin>
162 | 				<plugin>
163 | 					<groupId>org.codehaus.mojo</groupId>
164 | 					<artifactId>cassandra-maven-plugin</artifactId>
165 | 					<version>1.2.1-1</version>
166 | 				</plugin>
167 | 
168 | 				<!--This plugin's configuration is used to store Eclipse m2e settings 
169 | 					only. It has no influence on the Maven build itself. -->
170 | 				<plugin>
171 | 					<groupId>org.eclipse.m2e</groupId>
172 | 					<artifactId>lifecycle-mapping</artifactId>
173 | 					<version>1.0.0</version>
174 | 					<configuration>
175 | 						<lifecycleMappingMetadata>
176 | 							<pluginExecutions>
177 | 								<pluginExecution>
178 | 									<pluginExecutionFilter>
179 | 										<groupId>
180 | 											org.codehaus.mojo
181 | 										</groupId>
182 | 										<artifactId>
183 | 											build-helper-maven-plugin
184 | 										</artifactId>
185 | 										<versionRange>
186 | 											[1.7,)
187 | 										</versionRange>
188 | 										<goals>
189 | 											<goal>
190 | 												reserve-network-port
191 | 											</goal>
192 | 										</goals>
193 | 									</pluginExecutionFilter>
194 | 									<action>
195 | 										<ignore></ignore>
196 | 									</action>
197 | 								</pluginExecution>
198 | 							</pluginExecutions>
199 | 						</lifecycleMappingMetadata>
200 | 					</configuration>
201 | 				</plugin>
202 | 			</plugins>
203 | 		</pluginManagement>
204 | 		<plugins>
205 | 			<plugin>
206 | 				<groupId>org.apache.felix</groupId>
207 | 				<artifactId>maven-bundle-plugin</artifactId>
208 | 				<extensions>true</extensions>
209 | 				<version>2.4.0</version>
210 | 				<!-- <executions> <execution> <id>bundle-manifest</id> <phase>process-classes</phase> 
211 | 					<goals> <goal>manifest</goal> </goals> </execution> </executions> -->
212 | 				<configuration>
213 | 					<instructions>
214 | 						<Bundle-SymbolicName>com.github.adejanovski.cassandra.jdbc</Bundle-SymbolicName>
215 | 						<Bundle-Version>${project.version}</Bundle-Version>
216 | 						<_include>-osgi.bnd</_include>
217 | 					</instructions>
218 | 					<supportedProjectTypes>
219 | 						<supportedProjectType>jar</supportedProjectType>
220 | 						<supportedProjectType>bundle</supportedProjectType>
221 | 						<supportedProjectType>pom</supportedProjectType>
222 | 					</supportedProjectTypes>
223 | 				</configuration>
224 | 			</plugin>
225 | 			<plugin>
226 | 				<artifactId>maven-compiler-plugin</artifactId>
227 | 				<configuration>
228 | 					<source>1.6</source>
229 | 					<target>1.6</target>
230 | 				</configuration>
231 | 			</plugin>
232 | 			<plugin>
233 | 				<artifactId>maven-surefire-plugin</artifactId>
234 | 				<configuration>
235 | 					<includes>
236 | 						<include>**/*UnitTest.class</include>
237 | 					</includes>
238 | 					<systemPropertyVariables>
239 | 						<cassandra.version>${cassandra.version}</cassandra.version>
240 | 						<ipprefix>${ipprefix}</ipprefix>
241 | 						<com.datastax.driver.NEW_NODE_DELAY_SECONDS>60</com.datastax.driver.NEW_NODE_DELAY_SECONDS>
242 | 					</systemPropertyVariables>
243 | 				</configuration>
244 | 			</plugin>
245 | 			<plugin>
246 | 				<artifactId>maven-failsafe-plugin</artifactId>
247 | 				<configuration>
248 | 					<includes>
249 | 						<include>**/*Test.class</include>
250 | 					</includes>
251 | 					<excludes>
252 | 						<exclude>**/*UnitTest.class</exclude>
253 | 					</excludes>
254 | 				</configuration>
255 | 				<executions>
256 | 					<execution>
257 | 						<goals>
258 | 							<goal>integration-test</goal>
259 | 							<goal>verify</goal>
260 | 						</goals>
261 | 					</execution>
262 | 				</executions>
263 | 			</plugin>
264 | 		</plugins>
265 | 	</build>
266 | 
267 | 	<profiles>
268 | 		<profile>
269 | 			<id>default</id>
270 | 			<properties>
271 | 				<env>default</env>
272 | 			</properties>
273 | 			<activation>
274 | 				<activeByDefault>true</activeByDefault>
275 | 			</activation>
276 | 			<build>
277 | 				<plugins>
278 | 					<plugin>
279 | 						<artifactId>maven-surefire-plugin</artifactId>
280 | 						<version>2.17</version>
281 | 						<configuration>
282 | 							<skip>true</skip>
283 | 						</configuration>
284 | 						<executions>
285 | 							<execution>
286 | 								<phase>test</phase>
287 | 								<goals>
288 | 									<goal>test</goal>
289 | 								</goals>
290 | 								<configuration>
291 | 									<skip>true</skip>
292 | 								</configuration>
293 | 							</execution>
294 | 						</executions>
295 | 					</plugin>
296 | 					<plugin>
297 | 						<artifactId>maven-compiler-plugin</artifactId>
298 | 						<configuration>
299 | 							<compilerArgument>-Xlint:all</compilerArgument>
300 | 							<showWarnings>true</showWarnings>
301 | 							<showDeprecation>true</showDeprecation>
302 | 						</configuration>
303 | 					</plugin>					
304 | 					<plugin>
305 | 						<groupId>org.apache.maven.plugins</groupId>
306 | 						<artifactId>maven-javadoc-plugin</artifactId>
307 | 						<configuration>
308 | 					      <additionalparam>-Xdoclint:none</additionalparam>
309 | 					    </configuration>
310 | 						<version>2.10.3</version>
311 | 					</plugin>					
312 | 
313 | 				</plugins>
314 | 			</build>
315 | 		</profile>
316 | 
317 | 		<profile>
318 | 			<id>release</id>
319 | 			<properties>
320 | 				<env>default</env>
321 | 			</properties>
322 | 			<build>
323 | 				<plugins>
324 | 					<plugin>
325 | 						<artifactId>maven-surefire-plugin</artifactId>
326 | 						<version>2.17</version>
327 | 						<configuration>
328 | 							<skip>true</skip>
329 | 						</configuration>
330 | 						<executions>
331 | 							<execution>
332 | 								<phase>test</phase>
333 | 								<goals>
334 | 									<goal>test</goal>
335 | 								</goals>
336 | 								<configuration>
337 | 									<skip>true</skip>
338 | 								</configuration>
339 | 							</execution>
340 | 						</executions>
341 | 					</plugin>
342 | 					<plugin>
343 | 						<artifactId>maven-compiler-plugin</artifactId>
344 | 						<configuration>
345 | 							<compilerArgument>-Xlint:all</compilerArgument>
346 | 							<showWarnings>true</showWarnings>
347 | 							<showDeprecation>true</showDeprecation>
348 | 						</configuration>
349 | 					</plugin>
350 | 					<plugin>
351 | 						<groupId>org.apache.maven.plugins</groupId>
352 | 						<artifactId>maven-javadoc-plugin</artifactId>
353 | 						<version>2.10.3</version>
354 | 						<configuration>
355 | 					      <additionalparam>-Xdoclint:none</additionalparam>
356 | 					    </configuration>
357 | 					</plugin>
358 | 					<plugin>
359 | 						<groupId>org.apache.maven.plugins</groupId>
360 | 						<artifactId>maven-source-plugin</artifactId>
361 | 						<version>2.4</version>
362 | 						<executions>
363 | 							<execution>
364 | 								<id>attach-sources</id>
365 | 								<phase>verify</phase>
366 | 								<goals>
367 | 									<goal>jar-no-fork</goal>
368 | 								</goals>
369 | 							</execution>
370 | 						</executions>
371 | 					</plugin>
372 | 					<plugin>
373 | 						<groupId>org.apache.maven.plugins</groupId>
374 | 						<artifactId>maven-gpg-plugin</artifactId>
375 | 						<executions>
376 | 							<execution>
377 | 								<id>sign-artifacts</id>
378 | 								<phase>verify</phase>
379 | 								<goals>
380 | 									<goal>sign</goal>
381 | 								</goals>
382 | 							</execution>
383 | 						</executions>
384 | 					</plugin>
385 | 
386 | 
387 | 				</plugins>
388 | 			</build>
389 | 		</profile>
390 | 
391 | 		<profile>
392 | 			<id>test</id>
393 | 			<properties>
394 | 				<env>default</env>
395 | 			</properties>
396 | 			<build>
397 | 				<plugins>
398 | 					<plugin>
399 | 						<groupId>org.apache.maven.plugins</groupId>
400 | 						<artifactId>maven-surefire-plugin</artifactId>
401 | 						<version>2.18.1</version>						
402 | 					</plugin>
403 | 				</plugins>
404 | 			</build>
405 | 		</profile>
406 | 	</profiles>
407 | 
408 | 	<scm>
409 | 		<connection>scm:git:git@github.com:adejanovski/cassandra-dcaware-failover.git</connection>
410 | 		<developerConnection>scm:git:git@github.com:adejanovski/cassandra-dcaware-failover.git</developerConnection>
411 | 		<url>https://github.com/com:adejanovski/cassandra-dcaware-failover</url>
412 | 		<tag>HEAD</tag>
413 | 	</scm>
414 | 
415 | </project>
416 | 


--------------------------------------------------------------------------------
/DCAwareFailoverRoundRobinPolicy/src/main/java/org/adejanovski/cassandra/policies/DCAwareFailoverRoundRobinPolicy.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  *      Copyright (C) 2012-2015 DataStax Inc.
  3 |  *
  4 |  *   Licensed under the Apache License, Version 2.0 (the "License");
  5 |  *   you may not use this file except in compliance with the License.
  6 |  *   You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  *   Unless required by applicable law or agreed to in writing, software
 11 |  *   distributed under the License is distributed on an "AS IS" BASIS,
 12 |  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  *   See the License for the specific language governing permissions and
 14 |  *   limitations under the License.
 15 |  */
 16 | package org.adejanovski.cassandra.policies;
 17 | 
 18 | import java.util.*;
 19 | import java.util.concurrent.ConcurrentHashMap;
 20 | import java.util.concurrent.ConcurrentMap;
 21 | import java.util.concurrent.CopyOnWriteArrayList;
 22 | import java.util.concurrent.ExecutionException;
 23 | import java.util.concurrent.TimeUnit;
 24 | import java.util.concurrent.TimeoutException;
 25 | import java.util.concurrent.atomic.AtomicBoolean;
 26 | import java.util.concurrent.atomic.AtomicInteger;
 27 | 
 28 | import com.google.common.base.Joiner;
 29 | import com.google.common.base.Preconditions;
 30 | import com.google.common.base.Strings;
 31 | import com.google.common.annotations.VisibleForTesting;
 32 | import com.google.common.collect.AbstractIterator;
 33 | 
 34 | import org.slf4j.Logger;
 35 | import org.slf4j.LoggerFactory;
 36 | 
 37 | import com.datastax.driver.core.Cluster;
 38 | import com.datastax.driver.core.Configuration;
 39 | import com.datastax.driver.core.ConsistencyLevel;
 40 | import com.datastax.driver.core.Host;
 41 | import com.datastax.driver.core.HostDistance;
 42 | import com.datastax.driver.core.Statement;
 43 | import com.datastax.driver.core.policies.CloseableLoadBalancingPolicy;
 44 | import com.datastax.driver.core.policies.LoadBalancingPolicy;
 45 | 
 46 | /**
 47 |  * A data-center aware Round-robin load balancing policy with DC failover
 48 |  * support.
 49 |  *
 50 |  * This policy provides round-robin queries over the node of the local data
 51 |  * center. It also includes in the query plans returned a configurable number of
 52 |  * hosts in the remote data centers, but those are always tried after the local
 53 |  * nodes. In other words, this policy guarantees that no host in a remote data
 54 |  * center will be queried unless no host in the local data center can be
 55 |  * reached.
 56 |  *
 57 |  * If used with a single data center, this policy is equivalent to the
 58 |  * RoundRobinPolicy, but its DC awareness incurs a
 59 |  * slight overhead so the RoundRobinPolicy could be
 60 |  * preferred to this policy in that case.
 61 |  *
 62 |  * On top of the DCAwareRoundRobinPolicy, this policy uses a one way switch in
 63 |  * case a defined number of nodes are down in the local DC. As stated, the
 64 |  * policy never switches back to the local DC in order to prevent
 65 |  * inconsistencies and give ops teams the ability to repair the local DC before
 66 |  * switching back manually.
 67 |  */
 68 | /**
 69 |  * @author adejanovski
 70 |  *
 71 |  */
 72 | /**
 73 |  * @author adejanovski
 74 |  *
 75 |  */
 76 | public class DCAwareFailoverRoundRobinPolicy implements LoadBalancingPolicy,
 77 | 		CloseableLoadBalancingPolicy {
 78 | 
 79 | 	private static final Logger logger = LoggerFactory
 80 | 			.getLogger(DCAwareFailoverRoundRobinPolicy.class);
 81 | 
 82 | 	/**
 83 |      * Returns a builder to create a new instance.
 84 |      *
 85 |      * @return the builder.
 86 |      */
 87 |     public static Builder builder() {
 88 |         return new Builder();
 89 |     }
 90 | 	
 91 | 	private final String UNSET = "";
 92 | 
 93 | 	private final ConcurrentMap<String, CopyOnWriteArrayList<Host>> perDcLiveHosts = new ConcurrentHashMap<String, CopyOnWriteArrayList<Host>>();
 94 | 	private final AtomicInteger index = new AtomicInteger();
 95 | 
 96 | 	volatile String localDc;
 97 | 	volatile String backupDc;
 98 | 
 99 | 	/**
100 | 	 * Current value of the switch threshold. if {@code hostDownSwitchThreshold}
101 | 	 * is lower than 0 then we must switch.
102 | 	 */
103 | 	private AtomicInteger hostDownSwitchThreshold = new AtomicInteger();
104 | 	
105 | 	/**
106 | 	 * Initial value of the switch threshold
107 | 	 */
108 | 	private final int initHostDownSwitchThreshold;
109 | 
110 | 	/**
111 | 	 * flag to test if the switch as occurred
112 | 	 */
113 | 	private AtomicBoolean switchedToBackupDc = new AtomicBoolean(false);
114 | 	
115 | 	/**
116 | 	 * Time at which the switch occurred
117 | 	 */
118 | 	private Date switchedToBackupDcAt;
119 | 
120 | 	/**
121 | 	 * Automatically switching back to local DC is possible after : downtime*{@code switchBackDelayFactor}
122 | 	 */
123 | 	private Float switchBackDelayFactor=(float)1000;
124 | 
125 | 	/**
126 | 	 * Downtime delay after which switching back cannot be automated (usually
127 | 	 * when hinted handoff window is reached) In seconds.
128 | 	 */
129 | 	private int noSwitchBackDowntimeDelay=0;
130 | 
131 | 	private Date localDcCameBackUpAt;
132 | 	private boolean switchBackCanNeverHappen=false;
133 | 
134 | 	private volatile Configuration configuration;
135 | 
136 | 	/**
137 | 	 * Creates a new datacenter aware failover round robin policy that uses a
138 | 	 * local data-center and a backup data-center. Switching to the backup DC is
139 | 	 * triggered automatically if local DC loses more than
140 | 	 * {@code hostDownSwitchThreshold} nodes. Switching back to local DC after
141 | 	 * going to backup will never happen automatically.
142 | 	 * @param localDc the local datacenter
143 | 	 * @param backupDc the backup datacenter
144 | 	 * @param hostDownSwitchThreshold how many nodes have to be down before switching
145 | 	 */
146 | 	public DCAwareFailoverRoundRobinPolicy(String localDc, String backupDc,
147 | 			int hostDownSwitchThreshold) {
148 | 
149 | 		this(localDc, backupDc, hostDownSwitchThreshold, (float) -1.0, 0);
150 | 
151 | 	}
152 | 
153 | 	/**
154 | 	 * Creates a new datacenter aware failover round robin policy that uses a
155 | 	 * local data-center and a backup data-center. Switching to the backup DC is
156 | 	 * triggered automatically if local DC loses more than
157 | 	 * {@code hostDownSwitchThreshold} nodes.
158 | 	 * The policy will switch back to the local DC if conditions are fulfilled : 
159 | 	 * - Downtime lasted less than noSwitchBackDowntimeDelay (hint window)
160 | 	 * - uptime since downtime happened is superior to downtime*switchBackDelayFactor (give
161 | 	 * 	 enough time for hints to be executed)
162 | 	 * 
163 | 	 * @param localDc the local datacenter
164 | 	 * @param backupDc the backup datacenter
165 | 	 * @param hostDownSwitchThreshold how many nodes have to be down before switching
166 | 	 * @param switchBackDelayFactor uptime since downtime happened is superior to downtime*switchBackDelayFactor
167 | 	 * @param noSwitchBackDowntimeDelay maximum downtime to authorize a back switch to local DC
168 | 	 */
169 | 	public DCAwareFailoverRoundRobinPolicy(String localDc, String backupDc,
170 | 			int hostDownSwitchThreshold, float switchBackDelayFactor,
171 | 			int noSwitchBackDowntimeDelay) {
172 | 		this.localDc = localDc == null ? UNSET : localDc;
173 | 		this.backupDc = backupDc == null ? UNSET : backupDc;
174 | 		this.hostDownSwitchThreshold = new AtomicInteger(hostDownSwitchThreshold);
175 | 		this.initHostDownSwitchThreshold = hostDownSwitchThreshold;
176 | 		this.switchBackDelayFactor = switchBackDelayFactor;
177 | 		this.noSwitchBackDowntimeDelay = noSwitchBackDowntimeDelay;
178 | 
179 | 	}
180 | 
181 | 	public void init(Cluster cluster, Collection<Host> hosts) {
182 | 		if (localDc != UNSET)
183 | 			logger.info(
184 | 					"Using provided data-center name '{}' for DCAwareFailoverRoundRobinPolicy",
185 | 					localDc);
186 | 
187 | 		this.configuration = cluster.getConfiguration();
188 | 
189 | 		ArrayList<String> notInLocalDC = new ArrayList<String>();
190 | 
191 | 		for (Host host : hosts) {
192 | 			String dc = dc(host);
193 | 
194 | 			logger.trace("node {} is in dc {}", host.getAddress().toString(), dc);
195 | 			// If the localDC was in "auto-discover" mode and it's the first
196 | 			// host for which we have a DC, use it.
197 | 			if (localDc == UNSET && dc != UNSET) {
198 | 				logger.info(
199 | 						"Using data-center name '{}' for DCAwareFailoverRoundRobinPolicy (if this is incorrect, please provide the correct datacenter name with DCAwareFailoverRoundRobinPolicy constructor)",
200 | 						dc);
201 | 				localDc = dc;
202 | 			} else if (!dc.equals(localDc) && !dc.equals(backupDc))
203 | 				notInLocalDC.add(String.format("%s (%s)", host.toString(), dc));
204 | 
205 | 			if (!dc.equals(localDc) && !dc.equals(backupDc))
206 | 				notInLocalDC.add(String.format("%s (%s)", host.toString(),
207 | 						host.getDatacenter()));
208 | 
209 | 			CopyOnWriteArrayList<Host> prev = perDcLiveHosts.get(dc);
210 | 			if (prev == null)
211 | 				perDcLiveHosts.put(dc, new CopyOnWriteArrayList<Host>(
212 | 						Collections.singletonList(host)));
213 | 			else
214 | 				prev.addIfAbsent(host);
215 | 		}
216 | 
217 | 		if (notInLocalDC.size() > 0) {
218 | 			String nonLocalHosts = Joiner.on(",").join(notInLocalDC);
219 | 			logger.warn(
220 | 					"Some contact points don't match local or backup data center. Local DC = {} - backup DC {}. Non-conforming contact points: {}",
221 | 					localDc, backupDc, nonLocalHosts);
222 | 		}
223 | 	}
224 | 
225 | 	private String dc(Host host) {
226 | 		String dc = host.getDatacenter();
227 | 		return dc == null ? localDc : dc;
228 | 	}
229 | 
230 | 	@SuppressWarnings("unchecked")
231 | 	private static CopyOnWriteArrayList<Host> cloneList(
232 | 			CopyOnWriteArrayList<Host> list) {
233 | 		return (CopyOnWriteArrayList<Host>) list.clone();
234 | 	}
235 | 
236 | 	/**
237 | 	 * Return the HostDistance for the provided host.
238 | 	 * 
239 | 	 * This policy consider nodes in the local datacenter as {@code LOCAL}. For
240 | 	 * each remote datacenter, it considers a configurable number of hosts as
241 | 	 * {@code REMOTE} and the rest is {@code IGNORED}.
242 | 	 * 
243 | 	 * To configure how many host in each remote datacenter is considered
244 | 	 * {@code REMOTE}.
245 | 	 *
246 | 	 * @param host
247 | 	 *            the host of which to return the distance of.
248 | 	 * @return the HostDistance to {@code host}.
249 | 	 */
250 | 	public HostDistance distance(Host host) {
251 | 		String dc = dc(host);
252 | 		// If the connection has switched to the backup DC and fulfills
253 | 		// the requirement for a back switch, make it happen.
254 | 		if(!switchBackCanNeverHappen){
255 | 			triggerBackSwitchIfNecessary();
256 | 		}
257 | 
258 | 		if (isLocal(dc)) {			
259 | 			return HostDistance.LOCAL;
260 | 		}
261 | 
262 | 		// Only hosts in local DC and backup DC can be considered remote
263 | 		if(dc(host).equals(localDc) || dc(host).equals(backupDc))
264 | 			return HostDistance.REMOTE;
265 | 		
266 | 		// All other hosts are ignored
267 | 		return HostDistance.IGNORED;
268 | 		
269 | 	}
270 | 
271 | 	/**
272 | 	 * Returns the hosts to use for a new query.
273 | 	 * 
274 | 	 * The returned plan will always try each known host in the local datacenter
275 | 	 * first, and then, if none of the local host is reachable, will try up to a
276 | 	 * configurable number of other host per remote datacenter. The order of the
277 | 	 * local node in the returned query plan will follow a Round-robin
278 | 	 * algorithm.
279 | 	 *
280 | 	 * @param loggedKeyspace
281 | 	 *            the keyspace currently logged in on for this query.
282 | 	 * @param statement
283 | 	 *            the query for which to build the plan.
284 | 	 * @return a new query plan, i.e. an iterator indicating which host to try
285 | 	 *         first for querying, which one to use as failover, etc...
286 | 	 */
287 | 	public Iterator<Host> newQueryPlan(String loggedKeyspace,
288 | 			final Statement statement) {
289 | 		String currentDc = localDc;
290 | 		if(!switchBackCanNeverHappen){
291 | 			triggerBackSwitchIfNecessary();
292 | 		}
293 | 
294 | 		if (switchedToBackupDc.get()) {
295 | 			currentDc = backupDc;
296 | 		}
297 | 		
298 | 		CopyOnWriteArrayList<Host> localLiveHosts = perDcLiveHosts.get(currentDc);
299 | 		final List<Host> hosts = localLiveHosts == null ? Collections.<Host> emptyList() : cloneList(localLiveHosts);
300 | 
301 | 		final int startIdx = index.getAndIncrement();
302 | 
303 | 		return new AbstractIterator<Host>() {
304 | 
305 | 			private int idx = startIdx;
306 | 			private int remainingLocal = hosts.size();
307 | 
308 | 			// For remote Dcs
309 | 			private Iterator<String> remoteDcs;
310 | 			private List<Host> currentDcHosts;
311 | 			private int currentDcRemaining;
312 | 
313 | 			@Override
314 | 			protected Host computeNext() {				
315 | 				if (remainingLocal > 0) {
316 | 					remainingLocal--;					
317 | 					int c = idx++ % hosts.size();
318 | 					if (c < 0) {
319 | 						c += hosts.size();
320 | 					}					
321 | 					return hosts.get(c);
322 | 				}
323 | 									
324 | 				return endOfData();
325 | 			}
326 | 		};
327 | 	}
328 | 
329 | 	public void onUp(Host host) {
330 | 
331 | 		String dc = dc(host);		
332 | 		if (dc.equals(localDc) && this.hostDownSwitchThreshold.get() < this.initHostDownSwitchThreshold
333 | 				) {
334 | 			// if a node comes backup in the local DC and we're not already
335 | 			// equal to the initial threshold, add one node to the
336 | 			// switch threshold
337 | 			// This can only happen if the switch didn't occur yet
338 | 			this.hostDownSwitchThreshold.incrementAndGet();
339 | 			updateLocalDcStatus();
340 | 		}
341 | 		// If the localDC was in "auto-discover" mode and it's the first host
342 | 		// for which we have a DC, use it.
343 | 		if (localDc == UNSET && dc != UNSET) {
344 | 			logger.info(
345 | 					"Using data-center name '{}' for DCAwareFailoverRoundRobinPolicy (if this is incorrect, please provide the correct datacenter name with DCAwareFailoverRoundRobinPolicy constructor)",
346 | 					dc);
347 | 			localDc = dc;
348 | 		}
349 | 
350 | 		CopyOnWriteArrayList<Host> dcHosts = perDcLiveHosts.get(dc);
351 | 		if (dcHosts == null) {
352 | 			CopyOnWriteArrayList<Host> newMap = new CopyOnWriteArrayList<Host>(Collections.singletonList(host));
353 | 			dcHosts = perDcLiveHosts.putIfAbsent(dc, newMap);
354 | 			// If we've successfully put our new host, we're good, otherwise
355 | 			// we've been beaten so continue
356 | 			if (dcHosts == null)
357 | 				return;
358 | 		}
359 | 		dcHosts.addIfAbsent(host);
360 | 	}
361 | 
362 | 	public void onSuspected(Host host) {
363 | 	}
364 | 
365 | 	public void onDown(Host host) {		
366 | 		if (dc(host).equals(localDc) && !switchedToBackupDc.get()) {
367 | 			// if a node goes down in the local DC remove one node to eventually
368 | 			// trigger the switch
369 | 			this.hostDownSwitchThreshold.decrementAndGet();
370 | 		}
371 | 		CopyOnWriteArrayList<Host> dcHosts = perDcLiveHosts.get(dc(host));
372 | 		if (dcHosts != null)
373 | 			dcHosts.remove(host);
374 | 
375 | 		if (this.hostDownSwitchThreshold.get() <= 0) {
376 | 			// Make sure localDc is not considered as being up
377 | 			localDcCameBackUpAt = null;
378 | 			if (!switchedToBackupDc.get()) {
379 | 				// if we lost as many nodes in the local dc as configured in the
380 | 				// threshold, switch to backup DC
381 | 				switchToBackup();
382 | 			}
383 | 		}
384 | 	}
385 | 
386 | 	public void onAdd(Host host) {
387 | 		onUp(host);
388 | 	}
389 | 
390 | 	public void onRemove(Host host) {
391 | 		onDown(host);
392 | 	}
393 | 
394 | 	public void close() {
395 | 		// nothing to do
396 | 	}
397 | 
398 | 	/**
399 | 	 * Perform switch to backup DC
400 | 	 */
401 | 	private void switchToBackup() {
402 | 		switchedToBackupDc.set(true);
403 | 		switchedToBackupDcAt = new Date();
404 | 		logger.warn(
405 | 				"Lost {} nodes in data-center '{}'. Switching to data-center '{}'",
406 | 				this.initHostDownSwitchThreshold, this.localDc, this.backupDc);
407 | 
408 | 	}
409 | 
410 | 	/**
411 | 	 * Perform switch back to local DC
412 | 	 */
413 | 	private void switchBackToLocal() {
414 | 		switchedToBackupDc.set(false);
415 | 		switchedToBackupDcAt = null;
416 | 		localDcCameBackUpAt = null;
417 | 		logger.warn(
418 | 				"Recovered enough nodes in data-center '{}'. Switching back since conditions are fulfilled",
419 | 				this.localDc);
420 | 
421 | 	}
422 | 
423 | 	/**
424 | 	 * Check if the cluster state fulfills requirements for switching back to
425 | 	 * local DC. Conditions to switch back : - the connection as already
426 | 	 * switched to backup DC - hostDownSwitchThreshold is > 0 - Enough time has
427 | 	 * passed for hinted handoff (currentTime - localDcCameBackUpAt) >
428 | 	 * (localDcCameBackUpAt - switchedToBackupDcAt)*switchBackDelayFactor -
429 | 	 * (localDcCameBackUpAt - switchedToBackupDcAt) < noSwitchBackDowntimeDelay
430 | 	 * 
431 | 	 * @return
432 | 	 */
433 | 	private boolean canSwitchBack() {		
434 | 		if ((localDcCameBackUpAt.getTime() - switchedToBackupDcAt.getTime()) < noSwitchBackDowntimeDelay * 1000) {
435 | 			if (switchedToBackupDc.get() && isLocalDcBackUp()) {
436 | 				logger.debug(
437 | 						"Local DC {} is up and has been down for {}s. Switch back will happen after {}s. Uptime = {}s ",
438 | 						localDc,
439 | 						(int) (getDowntimeDuration() / 1000),
440 | 						(int) (getDowntimeDuration() * switchBackDelayFactor / 1000),
441 | 						(getUptimeDuration()) / 1000);
442 | 				
443 | 				return (hostDownSwitchThreshold.get() > 0)
444 | 						&& (getUptimeDuration() > getDowntimeDuration() * switchBackDelayFactor)
445 | 						&& getDowntimeDuration() < noSwitchBackDowntimeDelay * 1000;
446 | 			}
447 | 		}else{
448 | 			// Downtime lasted more than the hinted handoff window
449 | 			// Switching back is now a manual operation
450 | 			logger.warn(
451 | 					"Local DC has been down for too long. Switch back will never happen.");
452 | 			switchBackCanNeverHappen=true;
453 | 		}
454 | 
455 | 		return false;
456 | 
457 | 	}
458 | 	
459 | 	/**
460 | 	 * returns the duration of the local DC downtime.
461 | 	 * @return
462 | 	 */
463 | 	private long getDowntimeDuration(){
464 | 		return localDcCameBackUpAt.getTime() - switchedToBackupDcAt.getTime();
465 | 	}
466 | 	
467 | 	/**
468 | 	 * get the uptime duration of local DC after outage.
469 | 	 * @return
470 | 	 */
471 | 	private long getUptimeDuration(){		
472 | 		return new Date().getTime() - localDcCameBackUpAt.getTime();
473 | 	}
474 | 	
475 | 	
476 | 
477 | 	private void updateLocalDcStatus() {
478 | 		if (switchedToBackupDc.get() && hostDownSwitchThreshold.get() > 0 && localDcCameBackUpAt == null) {
479 | 			localDcCameBackUpAt = new Date();
480 | 		}
481 | 	}
482 | 
483 | 	/**
484 | 	 * Test if local DC has enough nodes to be considered alive
485 | 	 * 
486 | 	 * @return
487 | 	 */
488 | 	private boolean isLocalDcBackUp() {
489 | 		return hostDownSwitchThreshold.get() > 0 && localDcCameBackUpAt != null;
490 | 	}
491 | 
492 | 	/**
493 | 	 * Test if a node is in the local DC (or in the backup DC and switch has
494 | 	 * occurred)
495 | 	 * 
496 | 	 * @param dc
497 | 	 * @return
498 | 	 */
499 | 	private boolean isLocal(String dc) {
500 | 		return dc == UNSET || (dc.equals(localDc) && !switchedToBackupDc.get())
501 | 				|| (dc.equals(backupDc) && switchedToBackupDc.get());
502 | 	}
503 | 
504 | 	/**
505 | 	 * Check if a switch as occurred and switching back to local DC is possible.
506 | 	 */
507 | 	public void triggerBackSwitchIfNecessary() {
508 | 		if (switchedToBackupDc.get() && localDcCameBackUpAt!=null && switchedToBackupDcAt!=null) {
509 | 			if (canSwitchBack()) {
510 | 				switchBackToLocal();
511 | 			}
512 | 		}
513 | 	}
514 | 	
515 | 	
516 | 	/**
517 |      * Helper class to build the policy.
518 |      */
519 |     public static class Builder {
520 |         private String localDc;
521 |         private String backupDc;        
522 |         private int hostDownSwitchThreshold;
523 |         private Float switchBackDelayFactor=(float)1000;
524 |     	private int noSwitchBackDowntimeDelay=0;
525 | 
526 |         /**
527 |          * Sets the name of the datacenter that will be considered "local" by the policy.
528 |          * 
529 |          * This must be the name as known by Cassandra (in other words, the name in that appears in
530 |          * {@code system.peers}, or in the output of admin tools like nodetool).
531 |          * 
532 |          * If this method isn't called, the policy will default to the datacenter of the first node
533 |          * connected to. This will always be ok if all the contact points use at {@code Cluster}
534 |          * creation are in the local data-center. Otherwise, you should provide the name yourself
535 |          * with this method.
536 |          *
537 |          * @param localDc the name of the datacenter. It should not be {@code null}.
538 |          * @return this builder.
539 |          */
540 |         public Builder withLocalDc(String localDc) {
541 |             Preconditions.checkArgument(!Strings.isNullOrEmpty(localDc),
542 |                 "localDc name can't be null or empty. If you want to let the policy autodetect the datacenter, don't call Builder.withLocalDC");
543 |             this.localDc = localDc;
544 |             return this;
545 |         }
546 |         
547 |         /**
548 |          * Sets the name of the datacenter that will be considered as "backup" by the policy.
549 |          * <p>
550 |          * This must be the name as known by Cassandra (in other words, the name in that appears in
551 |          * {@code system.peers}, or in the output of admin tools like nodetool).
552 |          * <p>
553 |          * If this method must be called, otherwise you should not use this policy.
554 |          *
555 |          * @param backupDc the name of the datacenter. It should not be {@code null}.
556 |          * @return this builder.
557 |          */
558 |         public Builder withBackupDc(String backupDc) {
559 |             Preconditions.checkArgument(!Strings.isNullOrEmpty(localDc),
560 |                 "backupDc name can't be null or empty.");
561 |             this.backupDc = backupDc;
562 |             return this;
563 |         }
564 | 
565 |         
566 |         /**
567 |          * Sets how many nodes must be down in the local DC before switching to backup.  
568 |          * 
569 |          * @param hostDownSwitchThreshold the number of nodes down before switching to the backup DC.
570 |          * @return this builder
571 |          */
572 |         public Builder withHostDownSwitchThreshold(int hostDownSwitchThreshold) {
573 |             this.hostDownSwitchThreshold = hostDownSwitchThreshold;
574 |             return this;
575 |         }
576 |         
577 |         /**
578 |          * Mandatory if you want to authorize switching back to local DC after downtime. 
579 |          * Allows enough time to pass so that hinted handoff can finish 
580 |          * (currentTime - localDcCameBackUpAt) &gt; (localDcCameBackUpAt - switchedToBackupDcAt)*switchBackDelayFactor 
581 |          * 
582 |          * @param switchBackDelayFactor times downtime has to be &lt;= uptime before switching back to local DC 
583 |          * @return this builder
584 |          */
585 |         public Builder withSwitchBackDelayFactor(float switchBackDelayFactor) {
586 |             this.switchBackDelayFactor = switchBackDelayFactor;
587 |             return this;
588 |         }
589 |         
590 |         /**
591 |          * Mandatory if you want to authorize switching back to local DC after downtime.
592 |          * Prevents switching back to local DC if downtime was longer than the provided value.
593 |          * Used to check if downtime didn't last more than the hinted handoff window (which requires repair).
594 |          * 
595 |          * @param noSwitchBackDowntimeDelay max time in seconds before switching back to local DC will be prevented.
596 |          * @return this builder
597 |          */
598 |         public Builder withNoSwitchBackDowntimeDelay(int noSwitchBackDowntimeDelay) {
599 |             this.noSwitchBackDowntimeDelay = noSwitchBackDowntimeDelay;
600 |             return this;
601 |         }
602 |         
603 |         
604 | 
605 |         /**
606 |          * Builds the policy configured by this builder.
607 |          *
608 |          * @return the policy.
609 |          */
610 |         public DCAwareFailoverRoundRobinPolicy build() {
611 |             return new DCAwareFailoverRoundRobinPolicy(localDc, backupDc, hostDownSwitchThreshold, switchBackDelayFactor, noSwitchBackDowntimeDelay);
612 |         }
613 |     }
614 | 
615 | }
616 | 


--------------------------------------------------------------------------------