17 | * Does not perform any retrys on the server side but instead propagates
18 | * serialization errors to the client for retry.
19 | */
20 | @Configuration
21 | @Profile(ProfileNames.RETRY_NONE)
22 | public class NoRetryConfig {
23 | @Autowired
24 | private Environment environment;
25 |
26 | @PostConstruct
27 | public void checkProfiles() {
28 | Assert.isTrue(!environment.acceptsProfiles(Profiles.of(ProfileNames.RETRY_SAVEPOINT)),
29 | "Cant have both RETRY_NONE and RETRY_SAVEPOINT");
30 | Assert.isTrue(!environment.acceptsProfiles(Profiles.of(ProfileNames.RETRY_DRIVER)),
31 | "Cant have both RETRY_NONE and RETRY_DRIVER");
32 | Assert.isTrue(!environment.acceptsProfiles(Profiles.of(ProfileNames.RETRY_CLIENT)),
33 | "Cant have both RETRY_NONE and RETRY_CLIENT");
34 | }
35 | }
36 |
37 |
--------------------------------------------------------------------------------
/deploy/common/main.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | core_util.sh
4 |
5 | case "$OSTYPE" in
6 | darwin*)
7 | rootdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
8 | selfname="$(basename "$(test -L "$0" && readlink "$0" || echo "$0")")"
9 | ;;
10 | *)
11 | rootdir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
12 | selfname="$(basename "$(readlink -f "${BASH_SOURCE[0]}")")"
13 | ;;
14 | esac
15 |
16 | fn_echo_header
17 | {
18 | echo -e "${lightblue}Cluster id:\t\t${default}$CLUSTER"
19 | echo -e "${lightblue}Node count:\t\t${default}$nodes"
20 | echo -e "${lightblue}CRDB nodes:\t\t${default}$crdbnodes"
21 | echo -e "${lightblue}CRDB version:\t\t${default}$releaseversion"
22 | echo -e "${lightblue}Client nodes:\t\t${default}${clients[*]}"
23 | echo -e "${lightblue}Cloud:\t\t${default}$cloud"
24 | echo -e "${lightblue}Machine types:\t\t${default}$machinetypes"
25 | echo -e "${lightblue}Zones:\t\t${default}$zones"
26 | } | column -s $'\t' -t
27 |
28 | if [ -z "${CLUSTER}" ]; then
29 | fn_echo_warning "No \$CLUSTER id variable set!"
30 | echo "Use: export CLUSTER='your-cluster-id'"
31 | exit 1
32 | fi
33 |
34 | if fn_prompt_yes_no "1/5: Create CRDB cluster?" Y; then
35 | 01_create_cluster.sh
36 | fi
37 |
38 | if fn_prompt_yes_no "2/5: Deploy Bank Servers?" Y; then
39 | 02_deploy_servers.sh
40 | fi
41 |
42 | if fn_prompt_yes_no "3/5: Deploy Bank Clients?" Y; then
43 | 03_deploy_clients.sh
44 | fi
45 |
46 | if fn_prompt_yes_no "4/5: Start Bank Servers?" Y; then
47 | 04_start_servers.sh
48 | fi
49 |
50 | fn_echo_info_nl "Done!"
--------------------------------------------------------------------------------
/bank-client/src/main/java/io/roach/bank/client/RegionProvider.java:
--------------------------------------------------------------------------------
1 | package io.roach.bank.client;
2 |
3 | import java.util.ArrayList;
4 | import java.util.List;
5 |
6 | import org.springframework.beans.factory.annotation.Autowired;
7 | import org.springframework.shell.CompletionContext;
8 | import org.springframework.shell.CompletionProposal;
9 | import org.springframework.shell.standard.ValueProvider;
10 | import org.springframework.util.StringUtils;
11 |
12 | import io.roach.bank.api.Region;
13 | import io.roach.bank.client.support.HypermediaClient;
14 |
15 | public class RegionProvider implements ValueProvider {
16 | @Autowired
17 | private HypermediaClient bankClient;
18 |
19 | @Override
20 | public List
22 | * See https://www.cockroachlabs.com/docs/transactions.html#transaction-retries
23 | */
24 | @Configuration
25 | @Profile(ProfileNames.RETRY_SAVEPOINT)
26 | public class SavepointRetryConfig {
27 | @Autowired
28 | private Environment environment;
29 |
30 | @PostConstruct
31 | public void checkProfiles() {
32 | Assert.isTrue(!environment.acceptsProfiles(Profiles.of(ProfileNames.JPA)),
33 | "Savepoints are not supported in JPA/Hibernate");
34 | Assert.isTrue(!environment.acceptsProfiles(Profiles.of(ProfileNames.RETRY_NONE)),
35 | "Cant have both RETRY_SAVEPOINT and RETRY_NONE");
36 | Assert.isTrue(!environment.acceptsProfiles(Profiles.of(ProfileNames.RETRY_DRIVER)),
37 | "Cant have both RETRY_SAVEPOINT and RETRY_DRIVER");
38 | Assert.isTrue(!environment.acceptsProfiles(Profiles.of(ProfileNames.RETRY_CLIENT)),
39 | "Cant have both RETRY_SAVEPOINT and RETRY_CLIENT");
40 | }
41 |
42 | @Bean
43 | public SavepointRetryAspect savepointTransactionAspect(PlatformTransactionManager transactionManager) {
44 | return new SavepointRetryAspect(transactionManager, "cockroach_restart");
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/bank-server/src/main/java/io/roach/bank/web/ViewModel.java:
--------------------------------------------------------------------------------
1 | package io.roach.bank.web;
2 |
3 | import java.util.ArrayList;
4 | import java.util.List;
5 |
6 | import io.roach.bank.api.Region;
7 |
8 | public class ViewModel {
9 | private int limit;
10 |
11 | private String gatewayRegion;
12 |
13 | private String primaryRegion;
14 |
15 | private String secondaryRegion;
16 |
17 | private String viewRegion;
18 |
19 | private boolean viewingGatewayRegion;
20 |
21 | private String randomFact;
22 |
23 | private final List accountSummary = bankClient.fromRoot()
38 | .follow(LinkRelations.withCurie(REPORTING_REL))
39 | .follow(LinkRelations.withCurie(ACCOUNT_SUMMARY_REL))
40 | .withTemplateParameters(parameters)
41 | .toEntity(List.class);
42 |
43 | accountSummary.getBody().forEach(item -> console.info("%s", item));
44 | }
45 |
46 | @ShellMethod(value = "Report transaction summary", key = {"report-transactions", "rt"})
47 | @ShellMethodAvailability(Constants.CONNECTED_CHECK)
48 | public void reportTransactions(@ShellOption(help = Constants.REGIONS_HELP,
49 | defaultValue = Constants.DEFAULT_REGION,
50 | valueProvider = RegionProvider.class) String region
51 | ) {
52 | final Map
transactionSummary = bankClient.fromRoot()
56 | .follow(LinkRelations.withCurie(REPORTING_REL))
57 | .follow(LinkRelations.withCurie(TRANSACTION_SUMMARY_REL))
58 | .withTemplateParameters(parameters)
59 | .toEntity(List.class);
60 |
61 | transactionSummary.getBody().forEach(item -> console.info("%s", item));
62 | }
63 | }
64 |
--------------------------------------------------------------------------------
/bank-server/src/main/java/io/roach/bank/config/ClientSideRetryConfig.java:
--------------------------------------------------------------------------------
1 | package io.roach.bank.config;
2 |
3 | import java.util.concurrent.TimeUnit;
4 |
5 | import org.springframework.beans.factory.annotation.Autowired;
6 | import org.springframework.context.annotation.Bean;
7 | import org.springframework.context.annotation.Configuration;
8 | import org.springframework.context.annotation.Profile;
9 | import org.springframework.core.env.Environment;
10 | import org.springframework.core.env.Profiles;
11 | import org.springframework.data.cockroachdb.aspect.TransactionRetryAspect;
12 | import org.springframework.util.Assert;
13 |
14 | import io.micrometer.core.instrument.Counter;
15 | import io.micrometer.core.instrument.MeterRegistry;
16 | import io.micrometer.core.instrument.Timer;
17 | import io.roach.bank.ProfileNames;
18 | import jakarta.annotation.PostConstruct;
19 |
20 | /**
21 | * Transaction management with retries and exponential backoff handled at JDBC Driver level.
22 | */
23 | @Configuration
24 | @Profile({ProfileNames.RETRY_CLIENT})
25 | public class ClientSideRetryConfig {
26 | @Autowired
27 | private Environment environment;
28 |
29 | @Autowired
30 | private MeterRegistry meterRegistry;
31 |
32 | private Counter retryEvents;
33 |
34 | private Counter retryCalls;
35 |
36 | private Timer retryTime;
37 |
38 | @PostConstruct
39 | public void checkProfiles() {
40 | Assert.isTrue(!environment.acceptsProfiles(Profiles.of(ProfileNames.RETRY_NONE)),
41 | "Cant have both RETRY_CLIENT and RETRY_NONE");
42 | Assert.isTrue(!environment.acceptsProfiles(Profiles.of(ProfileNames.RETRY_SAVEPOINT)),
43 | "Cant have both RETRY_CLIENT and RETRY_SAVEPOINT");
44 | Assert.isTrue(!environment.acceptsProfiles(Profiles.of(ProfileNames.RETRY_DRIVER)),
45 | "Cant have both RETRY_CLIENT and RETRY_DRIVER");
46 |
47 | this.retryEvents = Counter.builder("roach.bank.retries.event")
48 | .description("Number of transient error events")
49 | .register(meterRegistry);
50 |
51 | this.retryCalls = Counter.builder("roach.bank.retries.call")
52 | .description("Number of retry calls (closed loop cycles)")
53 | .register(meterRegistry);
54 |
55 | this.retryTime = Timer.builder("roach.bank.retries.time")
56 | .description("Time spent in retry wait loops")
57 | .register(meterRegistry);
58 | }
59 |
60 | @Bean
61 | public TransactionRetryAspect transactionRetryAspect() {
62 | TransactionRetryAspect retryAspect = new TransactionRetryAspect();
63 | retryAspect.setRetryEventConsumer(retryEvent -> {
64 | this.retryEvents.increment(1);
65 | this.retryCalls.increment(retryEvent.getNumCalls());
66 | this.retryTime.record(retryEvent.getElapsedTime().toMillis(), TimeUnit.MILLISECONDS);
67 | });
68 | return retryAspect;
69 | }
70 | }
71 |
--------------------------------------------------------------------------------
/bank-client/src/main/java/io/roach/bank/client/config/HypermediaConfig.java:
--------------------------------------------------------------------------------
1 | package io.roach.bank.client.config;
2 |
3 | import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
4 | import org.apache.hc.client5.http.impl.classic.HttpClients;
5 | import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
6 | import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
7 | import org.apache.hc.core5.http.io.SocketConfig;
8 | import org.apache.hc.core5.pool.PoolConcurrencyPolicy;
9 | import org.apache.hc.core5.pool.PoolReusePolicy;
10 | import org.apache.hc.core5.util.Timeout;
11 | import org.springframework.beans.factory.annotation.Value;
12 | import org.springframework.boot.web.client.RestTemplateBuilder;
13 | import org.springframework.boot.web.client.RestTemplateCustomizer;
14 | import org.springframework.context.annotation.Bean;
15 | import org.springframework.context.annotation.Configuration;
16 | import org.springframework.hateoas.config.EnableHypermediaSupport;
17 | import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
18 | import org.springframework.web.client.RestTemplate;
19 |
20 | import io.roach.bank.client.support.HypermediaClient;
21 |
22 | @Configuration
23 | @EnableHypermediaSupport(type = {
24 | EnableHypermediaSupport.HypermediaType.HAL_FORMS, EnableHypermediaSupport.HypermediaType.HAL
25 | })
26 | public class HypermediaConfig implements RestTemplateCustomizer {
27 | @Value("${roachbank.http.maxTotal}")
28 | private int maxTotal;
29 |
30 | @Value("${roachbank.http.maxConnPerRoute}")
31 | private int maxConnPerRoute;
32 |
33 | @Bean
34 | public RestTemplate restTemplate(RestTemplateBuilder builder) {
35 | return builder.build();
36 | }
37 |
38 | @Override
39 | public void customize(RestTemplate restTemplate) {
40 | if (maxConnPerRoute <= 0 || maxTotal <= 0) {
41 | maxConnPerRoute = Runtime.getRuntime().availableProcessors() * 8;
42 | maxTotal = maxConnPerRoute * 2;
43 | }
44 |
45 | PoolingHttpClientConnectionManager connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
46 | .setDefaultSocketConfig(SocketConfig.custom()
47 | .setSoTimeout(Timeout.ofMinutes(1))
48 | .build())
49 | .setPoolConcurrencyPolicy(PoolConcurrencyPolicy.STRICT)
50 | .setConnPoolPolicy(PoolReusePolicy.LIFO)
51 | .setMaxConnTotal(maxTotal)
52 | .setMaxConnPerRoute(maxConnPerRoute)
53 | .build();
54 |
55 | CloseableHttpClient client = HttpClients.custom()
56 | .setConnectionManager(connectionManager)
57 | .build();
58 |
59 | restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory(client));
60 | }
61 |
62 | @Bean
63 | public HypermediaClient restCommands(RestTemplate restTemplate) {
64 | return new HypermediaClient(restTemplate);
65 | }
66 | }
67 |
--------------------------------------------------------------------------------
/bank-api/src/main/java/io/roach/bank/api/LinkRelations.java:
--------------------------------------------------------------------------------
1 | package io.roach.bank.api;
2 |
3 | /**
4 | * Defines domain specific web constants such as link relation names and resource names.
5 | */
6 | public abstract class LinkRelations {
7 | // Transaction history rels
8 | public static final String TRANSACTION_REL = "transaction";
9 |
10 | public static final String TRANSACTION_LIST_REL = "transaction-list";
11 |
12 | public static final String TRANSACTION_ITEM_REL = "transaction-item";
13 |
14 | public static final String TRANSACTION_ITEMS_REL = "transaction-item-list";
15 |
16 | public static final String TRANSFER_FORM_REL = "transfer-form";
17 |
18 | // Account rels
19 | public static final String ACCOUNT_REL = "account";
20 |
21 | public static final String ACCOUNT_LIST_REL = "account-list";
22 |
23 | public static final String ACCOUNT_TOP = "account-top";
24 |
25 | public static final String ACCOUNT_ONE_FORM_REL = "account-form";
26 |
27 | public static final String ACCOUNT_BATCH_FORM_REL = "account-batch";
28 |
29 | public static final String ACCOUNT_BALANCE_REL = "account-balance";
30 |
31 | public static final String ACCOUNT_BALANCE_SNAPSHOT_REL = "account-balance-snapshot";
32 |
33 | // Reporting link relations
34 |
35 | public static final String REPORTING_REL = "reporting";
36 |
37 | public static final String ACCOUNT_SUMMARY_REL = "account-summary";
38 |
39 | public static final String TRANSACTION_SUMMARY_REL = "transaction-summary";
40 |
41 | // Meta
42 |
43 | public static final String REGION_REL = "region";
44 |
45 | public static final String REGION_LIST_REL = "region-list";
46 |
47 | public static final String CITY_LIST_REL = "city-list";
48 |
49 | public static final String GATEWAY_REGION_REL = "gateway-region";
50 |
51 |
52 | // Generic context-scoped link relations
53 |
54 | public static final String OPEN_REL = "open";
55 |
56 | public static final String CLOSE_REL = "close";
57 |
58 | // Admin
59 |
60 | public static final String ADMIN_REL = "admin";
61 |
62 | public static final String POOL_SIZE_REL = "pool-size";
63 |
64 | public static final String POOL_CONFIG_REL = "pool-config";
65 |
66 | public static final String TOGGLE_TRACE_LOG = "toggle-tracelog";
67 |
68 | public static final String ACTUATOR_REL = "actuator";
69 |
70 | public static final String CONFIG_INDEX_REL = "config";
71 |
72 | public static final String CONFIG_REGION_REL = "config-region";
73 |
74 | public static final String CONFIG_MULTI_REGION_REL = "config-multiregion";
75 |
76 |
77 | // IANA standard link relations:
78 | // http://www.iana.org/assignments/link-relations/link-relations.xhtml
79 |
80 | public static final String CURIE_NAMESPACE = "roachbank";
81 |
82 | public static final String CURIE_PREFIX = CURIE_NAMESPACE + ":";
83 |
84 | private LinkRelations() {
85 | }
86 |
87 | public static String withCurie(String rel) {
88 | return CURIE_NAMESPACE + ":" + rel;
89 | }
90 | }
91 |
--------------------------------------------------------------------------------
/deploy/common/01_create_cluster.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | fn_create_cluster() {
4 | if [ "${cloud}" = "aws" ]; then
5 | echo roachprod create $CLUSTER --clouds=aws --aws-machine-type-ssd=${machinetypes} --geo --local-ssd --nodes=${nodes} --aws-zones=${zones}
6 | fn_failcheck roachprod create $CLUSTER --clouds=aws --aws-machine-type-ssd=${machinetypes} --geo --local-ssd --nodes=${nodes} --aws-zones=${zones} --aws-profile crl-revenue --aws-config ~/rev.json
7 | elif [ "${cloud}" = "gce" ]; then
8 | echo roachprod create $CLUSTER --clouds=gce --gce-machine-type=${machinetypes} --geo --local-ssd --nodes=${nodes} --gce-zones=${zones}
9 | fn_failcheck roachprod create $CLUSTER --clouds=gce --gce-machine-type=${machinetypes} --geo --local-ssd --nodes=${nodes} --gce-zones=${zones} --aws-profile crl-revenue --aws-config ~/rev.json
10 | else
11 | echo roachprod create $CLUSTER --clouds=azure --azure-machine-type=${machinetypes} --geo --local-ssd --nodes=${nodes} --azure-locations=${zones}
12 | fn_failcheck roachprod create $CLUSTER --clouds=azure --azure-machine-type=${machinetypes} --geo --local-ssd --nodes=${nodes} --azure-locations=${zones} --aws-profile crl-revenue --aws-config ~/rev.json
13 | fi
14 | }
15 |
16 | fn_stage_cluster() {
17 | fn_echo_info_nl "Stage binaries $releaseversion"
18 | fn_failcheck roachprod stage $CLUSTER release $releaseversion
19 | }
20 |
21 | fn_start_cluster() {
22 | fn_echo_info_nl "Start CockroachDB nodes $crdbnodes"
23 | fn_failcheck roachprod start $CLUSTER:$crdbnodes
24 | fn_failcheck roachprod admin --open --ips $CLUSTER:1
25 | }
26 |
27 | fn_stage_lb() {
28 | i=0;
29 | for c in "${clients[@]}"
30 | do
31 | region=${regions[$i]}
32 | i=($i+1)
33 |
34 | fn_echo_info_nl "Stage client ${CLUSTER}:$c"
35 |
36 | fn_failcheck roachprod run ${CLUSTER}:$c 'sudo apt-get -qq update'
37 | fn_failcheck roachprod run ${CLUSTER}:$c 'sudo apt-get -qq install -y openjdk-17-jre-headless htop dstat haproxy'
38 | fn_failcheck roachprod run ${CLUSTER}:$c "./cockroach gen haproxy --insecure --host $(roachprod ip $CLUSTER:1 --external) --locality=region=$region"
39 | fn_failcheck roachprod run ${CLUSTER}:$c 'nohup haproxy -f haproxy.cfg > /dev/null 2>&1 &'
40 | done
41 | }
42 |
43 | fn_create_db() {
44 | fn_echo_info_nl "Creating database via $CLUSTER:1"
45 |
46 | fn_failcheck roachprod run $CLUSTER:1 <