├── src ├── main │ ├── resources │ │ └── application.properties │ └── java │ │ └── com │ │ └── industrieit │ │ └── ledger │ │ └── clientledger │ │ └── web │ │ ├── exception │ │ ├── MessageDetail.java │ │ ├── ServiceException.java │ │ └── LedgerServiceErrorMessage.java │ │ ├── model │ │ ├── ledger │ │ │ └── Type.java │ │ └── request │ │ │ └── RequestEnvelop.java │ │ ├── Application.java │ │ ├── entity │ │ └── TransactionEvent.java │ │ ├── config │ │ └── LedgerConfig.java │ │ └── controller │ │ └── TransactionController.java └── test │ └── java │ └── com │ └── industrieit │ └── ledger │ └── clientledger │ └── web │ ├── ApplicationTest.java │ └── controller │ └── TransactionControllerTest.java ├── stress ├── kafka.sh ├── e2e.sh ├── p2p-request.json ├── back-up.sh ├── top-up.sh ├── p2p.sh └── create-account.sh ├── .gitignore ├── Dockerfile ├── docker-compose.yml ├── README.md └── pom.xml /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=3003 2 | 3 | -------------------------------------------------------------------------------- /stress/kafka.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | zkServer start 3 | kafka-server-start /usr/local/etc/kafka/server.properties 4 | 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | **/.DS_Store 3 | target 4 | **/*.iml 5 | stress/metrics.json 6 | stress/plot.html 7 | stress/results.bin 8 | transaction-logs/ 9 | -------------------------------------------------------------------------------- /stress/e2e.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | #Run an end to end flow, which create accounts, top up the payer, p2p from payer to payee, and then snapshot on all accounts 3 | ./create-account.sh 4 | ./top-up.sh 5 | ./p2p.sh 6 | ./back-up.sh 7 | ./p2p.sh 8 | 9 | -------------------------------------------------------------------------------- /src/main/java/com/industrieit/ledger/clientledger/web/exception/MessageDetail.java: -------------------------------------------------------------------------------- 1 | package com.industrieit.ledger.clientledger.web.exception; 2 | 3 | public interface MessageDetail { 4 | 5 | String getCode(); 6 | 7 | String getMessageKey(); 8 | 9 | } 10 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:11 2 | RUN apt-get update && apt-get install -y maven 3 | COPY . /project 4 | RUN cd /project && mvn package 5 | EXPOSE 8080 6 | ENTRYPOINT ["java", "-jar", "-Dspring.profiles.active=docker", "/project/target/client-ledger-service-1.0.0-SNAPSHOT.jar"] 7 | -------------------------------------------------------------------------------- /stress/p2p-request.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "p2p", 3 | "request": { 4 | "currency": "USD", 5 | "amount": 100, 6 | "fee": 10, 7 | "tax": 5, 8 | "fromCustomerAccount": "12345", 9 | "toCustomerAccount": "23456", 10 | "feeAccount": "34567", 11 | "taxAccount": "45678" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2.1" 2 | 3 | networks: 4 | client-ledger-core-db_test: 5 | external: true 6 | 7 | services: 8 | app: 9 | build: 10 | context: . 11 | dockerfile: Dockerfile 12 | ports: 13 | - "3003:3003" 14 | networks: 15 | - client-ledger-core-db_test 16 | 17 | 18 | -------------------------------------------------------------------------------- /stress/back-up.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | #Backup, have no effect on Maria Core, but trigger save to disk on Redis Core 3 | GREEN='\033[0;32m' 4 | NC='\033[0m' # No Color 5 | echo -e "${GREEN}Top Up Payer Account with USD 1,000,000${NC}" 6 | curl -X POST \ 7 | http://localhost:3003/transaction/event \ 8 | -H 'Content-Type: application/json' \ 9 | -d '{ 10 | "type" : "back-up", 11 | "request" : {} 12 | }' 13 | echo '\n' 14 | -------------------------------------------------------------------------------- /stress/top-up.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | #Top up the account, balance the entry using a settlement account 3 | GREEN='\033[0;32m' 4 | NC='\033[0m' # No Color 5 | echo -e "${GREEN}Top Up Payer Account with USD 1,000,000${NC}" 6 | curl -X POST \ 7 | http://localhost:3003/transaction/event \ 8 | -H 'Content-Type: application/json' \ 9 | -d '{ 10 | "type" : "top-up", 11 | "request" : { 12 | "currency": "USD", 13 | "amount": 1000000, 14 | "topUpAccount" : "12345", 15 | "settlementAccount" : "56789" 16 | } 17 | }' 18 | echo '\n' 19 | 20 | -------------------------------------------------------------------------------- /stress/p2p.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | #Pay from payer to payee account at a rate of 100/s to try to break the system 3 | GREEN='\033[0;32m' 4 | NC='\033[0m' # No Color 5 | echo -e "${GREEN}P2P USD 100 from Payer to Payee, for 5,000 times at a rate of 1000/s${NC}" 6 | echo "POST http://localhost:3003/transaction/event 7 | Content-Type: application/json 8 | @./p2p-request.json 9 | "| vegeta attack -duration=5s -rate=1000 | tee results.bin | vegeta report 10 | vegeta report -type=json results.bin > metrics.json 11 | cat results.bin | vegeta plot > plot.html 12 | cat results.bin | vegeta report -type="hist[0,100ms,200ms,300ms]" 13 | -------------------------------------------------------------------------------- /src/main/java/com/industrieit/ledger/clientledger/web/model/ledger/Type.java: -------------------------------------------------------------------------------- 1 | package com.industrieit.ledger.clientledger.web.model.ledger; 2 | 3 | 4 | import com.industrieit.ledger.clientledger.web.entity.TransactionEvent; 5 | 6 | /** 7 | * Type of {@link TransactionEvent} allowed for the Ledger to process 8 | */ 9 | public enum Type { 10 | P2P("p2p"), 11 | CREATE_ACCOUNT("create-account"), 12 | TOP_UP("top-up"), 13 | BACK_UP("back-up"); 14 | 15 | private final String text; 16 | 17 | Type(final String text) { 18 | this.text = text; 19 | } 20 | 21 | @Override 22 | public String toString() { 23 | return text; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/test/java/com/industrieit/ledger/clientledger/web/ApplicationTest.java: -------------------------------------------------------------------------------- 1 | package com.industrieit.ledger.clientledger.web; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class ApplicationTest { 7 | 8 | @Test 9 | public void testRun() { 10 | Application application = new Application(); 11 | application.run(); 12 | } 13 | 14 | @Test 15 | public void testRunWithParam() { 16 | try { 17 | Application application = new Application(); 18 | application.run("exitcode"); 19 | } catch (Application.ExitException e) { 20 | Assert.assertEquals(10, e.getExitCode()); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/industrieit/ledger/clientledger/web/exception/ServiceException.java: -------------------------------------------------------------------------------- 1 | package com.industrieit.ledger.clientledger.web.exception; 2 | 3 | 4 | public class ServiceException extends RuntimeException { 5 | 6 | private String errorCode; 7 | 8 | private String errorMessage; 9 | 10 | public ServiceException(MessageDetail messageDetail) { 11 | super(messageDetail.getMessageKey()); 12 | this.errorCode = messageDetail.getCode(); 13 | this.errorMessage = messageDetail.getMessageKey(); 14 | } 15 | 16 | 17 | public String getErrorCode() { 18 | return errorCode; 19 | } 20 | 21 | public String getErrorMessage() { 22 | return errorMessage; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/industrieit/ledger/clientledger/web/exception/LedgerServiceErrorMessage.java: -------------------------------------------------------------------------------- 1 | package com.industrieit.ledger.clientledger.web.exception; 2 | 3 | 4 | import com.industrieit.ledger.clientledger.web.entity.TransactionEvent; 5 | import com.industrieit.ledger.clientledger.web.model.request.RequestEnvelop; 6 | 7 | /** 8 | * Runtime transaction thrown before {@link RequestEnvelop} is accepted 9 | * No {@link TransactionEvent} will be created and enqueued when the exception is thrown 10 | */ 11 | public enum LedgerServiceErrorMessage implements MessageDetail { 12 | TYPE_NOT_SUPPORTED("LDR-1001", "Transaction Type Not Supported"), 13 | REQUEST_UNREADABLE("LDR-1002", "Transaction Request Unreadable"), 14 | 15 | ; 16 | 17 | private String code; 18 | private String messageKey; 19 | 20 | LedgerServiceErrorMessage(String code, String messageKey) { 21 | this.code = code; 22 | this.messageKey = messageKey; 23 | } 24 | 25 | public String getCode() { 26 | return code; 27 | } 28 | 29 | public String getMessageKey() { 30 | return messageKey; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/industrieit/ledger/clientledger/web/Application.java: -------------------------------------------------------------------------------- 1 | package com.industrieit.ledger.clientledger.web; 2 | 3 | 4 | import org.springframework.boot.CommandLineRunner; 5 | import org.springframework.boot.ExitCodeGenerator; 6 | import org.springframework.boot.SpringApplication; 7 | import org.springframework.boot.autoconfigure.SpringBootApplication; 8 | 9 | 10 | 11 | @SpringBootApplication 12 | public class Application implements CommandLineRunner { 13 | 14 | @Override 15 | public void run(String... arg0) { 16 | if (arg0.length > 0 && arg0[0].equals("exitcode")) { 17 | throw new ExitException(); 18 | } 19 | } 20 | 21 | public static void main(String[] args) { 22 | new SpringApplication(Application.class).run(args); 23 | } 24 | 25 | /** 26 | * Exit Exception on command line args of exitcode 27 | */ 28 | public static class ExitException extends RuntimeException implements ExitCodeGenerator { 29 | private static final long serialVersionUID = 1L; 30 | 31 | @Override 32 | public int getExitCode() { 33 | return 10; 34 | } 35 | 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/industrieit/ledger/clientledger/web/model/request/RequestEnvelop.java: -------------------------------------------------------------------------------- 1 | package com.industrieit.ledger.clientledger.web.model.request; 2 | 3 | 4 | import com.fasterxml.jackson.databind.JsonNode; 5 | import com.industrieit.ledger.clientledger.web.entity.TransactionEvent; 6 | import com.industrieit.ledger.clientledger.web.model.ledger.Type; 7 | 8 | /** 9 | * Envelop object which maps to request body for creating {@link TransactionEvent} 10 | */ 11 | public class RequestEnvelop { 12 | 13 | private String id; 14 | 15 | /** 16 | * @return a type which must be within the range of {@link Type} to be accepted 17 | */ 18 | public String getType() { 19 | return type; 20 | } 21 | 22 | public void setType(String type) { 23 | this.type = type; 24 | } 25 | 26 | private String type; 27 | 28 | private JsonNode request; 29 | 30 | /** 31 | * @return an important field for idempotency, must be generated from the client using {@link java.util.UUID} when in production 32 | */ 33 | public String getId() { 34 | return id; 35 | } 36 | 37 | public void setId(String id) { 38 | this.id = id; 39 | } 40 | 41 | /** 42 | * @return any JSON object which is the payload for creating {@link TransactionEvent} 43 | */ 44 | public JsonNode getRequest() { 45 | return request; 46 | } 47 | 48 | public void setRequest(JsonNode request) { 49 | this.request = request; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/industrieit/ledger/clientledger/web/entity/TransactionEvent.java: -------------------------------------------------------------------------------- 1 | package com.industrieit.ledger.clientledger.web.entity; 2 | 3 | import com.industrieit.ledger.clientledger.web.model.ledger.Type; 4 | 5 | /** 6 | * Entity which represents an accepted and enqueued high-level transaction, fully packed into a self-contained event 7 | * {@link TransactionEvent} can be consumed by consumer 8 | * On consumption, exactly one Transaction Result will be produced and persisted. 9 | * The full enqueued list of {@link TransactionEvent}, in a strict serial order, will form the basis of Event Sourcing. 10 | * Event sourcing allows the full state of the ledger be replayed, on any platform and infrastructure, with any processors. 11 | * This allows in-memory processing and reliable recovery from crash. 12 | */ 13 | 14 | public class TransactionEvent { 15 | private String id; 16 | private String type; 17 | private String request; 18 | 19 | /** 20 | * @return id which uniquely identify this transaction event. 21 | */ 22 | public String getId() { 23 | return id; 24 | } 25 | 26 | public void setId(String id) { 27 | this.id = id; 28 | } 29 | 30 | /** 31 | * @return payload of the request, usually a JSON string 32 | */ 33 | public String getRequest() { 34 | return request; 35 | } 36 | 37 | public void setRequest(String request) { 38 | this.request = request; 39 | } 40 | 41 | 42 | /** 43 | * @return type as defined in {@link Type} which calls for correct processor 44 | */ 45 | public String getType() { 46 | return type; 47 | } 48 | 49 | public void setType(String type) { 50 | this.type = type; 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/industrieit/ledger/clientledger/web/config/LedgerConfig.java: -------------------------------------------------------------------------------- 1 | package com.industrieit.ledger.clientledger.web.config; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.industrieit.ledger.clientledger.web.entity.TransactionEvent; 5 | import org.apache.kafka.clients.producer.ProducerConfig; 6 | import org.apache.kafka.common.serialization.StringSerializer; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.context.annotation.Bean; 10 | import org.springframework.context.annotation.Configuration; 11 | import org.springframework.kafka.core.DefaultKafkaProducerFactory; 12 | import org.springframework.kafka.core.KafkaTemplate; 13 | import org.springframework.kafka.core.ProducerFactory; 14 | import org.springframework.kafka.support.serializer.JsonSerializer; 15 | 16 | import java.util.HashMap; 17 | import java.util.Map; 18 | 19 | /** 20 | * Centralized place to inject beans 21 | */ 22 | @Configuration 23 | public class LedgerConfig { 24 | /** 25 | * @return {@link Logger} for standardized logging 26 | */ 27 | @Bean 28 | public Logger logger() { 29 | return LoggerFactory.getLogger("com.industrieit.dragon.clientledger.web"); 30 | } 31 | 32 | /** 33 | * @return {@link ObjectMapper} for JSON serialization, as Ledger is JSON-based. 34 | */ 35 | @Bean 36 | public ObjectMapper objectMapper() { 37 | return new ObjectMapper(); 38 | } 39 | 40 | @Bean 41 | public ProducerFactory producerFactoryForEvent() { 42 | Map config = new HashMap<>(); 43 | 44 | config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "kafka:9092"); 45 | config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); 46 | config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class); 47 | 48 | return new DefaultKafkaProducerFactory<>(config); 49 | } 50 | 51 | 52 | @Bean 53 | public KafkaTemplate kafkaTemplateForEvent() { 54 | return new KafkaTemplate<>(producerFactoryForEvent()); 55 | } 56 | 57 | } 58 | 59 | -------------------------------------------------------------------------------- /stress/create-account.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | #Create Payer, Payee, Fee, Tax and Settlement Account 3 | #To support additional features and transactions, create more accounts 4 | GREEN='\033[0;32m' 5 | NC='\033[0m' # No Color 6 | echo -e "${GREEN}Creating Payer Account${NC}" 7 | curl -X POST \ 8 | http://localhost:3003/transaction/event \ 9 | -H 'Content-Type: application/json' \ 10 | -d '{ 11 | "type" : "create-account", 12 | "request" : { 13 | "id" : "12345", 14 | "currency" :"USD", 15 | "accountName" : "Andrew", 16 | "accountGroup" : "Customer" 17 | } 18 | }' 19 | echo '\n' 20 | echo -e "${GREEN}Creating Payee Account${NC}" 21 | curl -X POST \ 22 | http://localhost:3003/transaction/event \ 23 | -H 'Content-Type: application/json' \ 24 | -d '{ 25 | "type" : "create-account", 26 | "request" : { 27 | "id" : "23456", 28 | "currency" :"USD", 29 | "accountName" : "Tim", 30 | "accountGroup" : "Customer" 31 | } 32 | }' 33 | echo '\n' 34 | echo -e "${GREEN}Creating Fee Account${NC}" 35 | curl -X POST \ 36 | http://localhost:3003/transaction/event \ 37 | -H 'Content-Type: application/json' \ 38 | -d '{ 39 | "type" : "create-account", 40 | "request" : { 41 | "id" : "34567", 42 | "currency" :"USD", 43 | "accountName" : "P2P Fee", 44 | "accountGroup" : "Fee" 45 | } 46 | }' 47 | echo '\n' 48 | echo -e "${GREEN}Creating Tax Account${NC}" 49 | curl -X POST \ 50 | http://localhost:3003/transaction/event \ 51 | -H 'Content-Type: application/json' \ 52 | -d '{ 53 | "type" : "create-account", 54 | "request" : { 55 | "id" : "45678", 56 | "currency" :"USD", 57 | "accountName" : "P2P Tax", 58 | "accountGroup" : "Tax" 59 | } 60 | }' 61 | echo '\n' 62 | echo -e "${GREEN}Creating Settlement Account${NC}" 63 | curl -X POST \ 64 | http://localhost:3003/transaction/event \ 65 | -H 'Content-Type: application/json' \ 66 | -d '{ 67 | "type" : "create-account", 68 | "request" : { 69 | "id" : "56789", 70 | "currency" :"USD", 71 | "accountName" : "Andrew'\''s settlement", 72 | "accountGroup" : "Settlement" 73 | } 74 | }' 75 | echo '\n' 76 | 77 | 78 | -------------------------------------------------------------------------------- /src/main/java/com/industrieit/ledger/clientledger/web/controller/TransactionController.java: -------------------------------------------------------------------------------- 1 | package com.industrieit.ledger.clientledger.web.controller; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.industrieit.ledger.clientledger.web.entity.TransactionEvent; 6 | import com.industrieit.ledger.clientledger.web.exception.LedgerServiceErrorMessage; 7 | import com.industrieit.ledger.clientledger.web.exception.ServiceException; 8 | import com.industrieit.ledger.clientledger.web.model.ledger.Type; 9 | import com.industrieit.ledger.clientledger.web.model.request.RequestEnvelop; 10 | import org.springframework.kafka.core.KafkaTemplate; 11 | import org.springframework.web.bind.annotation.*; 12 | 13 | import java.util.UUID; 14 | 15 | /** 16 | * REST Controller which is exclusively allowed to POST on the Ledger through creating and enqueuing {@link TransactionEvent} 17 | */ 18 | @RestController 19 | @RequestMapping("/transaction") 20 | public class TransactionController { 21 | 22 | private final ObjectMapper objectMapper; 23 | private final KafkaTemplate kafkaTemplate; 24 | public static final String TOPIC = "Transaction_Event"; 25 | 26 | 27 | 28 | public TransactionController( 29 | ObjectMapper objectMapper, 30 | KafkaTemplate kafkaTemplate) { 31 | this.objectMapper = objectMapper; 32 | this.kafkaTemplate = kafkaTemplate; 33 | } 34 | 35 | /** 36 | * Create and Enqueue one {@link TransactionEvent} onto the Blocking Queue for consumer to consume. 37 | * Raise {@link ServiceException} if unable to read the request JSON body, or the request type is not supported by {@link Type} 38 | * 39 | * @param requestEnvelop request for a transaction which mutates the Ledger's state 40 | * @return one {@link TransactionEvent} either created and enqueued, or already in the queue 41 | */ 42 | @PostMapping(value = "/event", 43 | produces = {"application/json"}, 44 | consumes = {"application/json"}) 45 | @ResponseBody 46 | public TransactionEvent queueTransaction(@RequestBody RequestEnvelop requestEnvelop) { 47 | TransactionEvent transactionEvent = new TransactionEvent(); 48 | try { 49 | transactionEvent.setRequest(objectMapper.writeValueAsString(requestEnvelop.getRequest())); 50 | } catch (JsonProcessingException e) { 51 | throw new ServiceException(LedgerServiceErrorMessage.REQUEST_UNREADABLE); 52 | } 53 | transactionEvent.setId(requestEnvelop.getId() == null ? UUID.randomUUID().toString() : requestEnvelop.getId()); 54 | for (Type type : Type.values()) { 55 | if (requestEnvelop.getType().equals(type.toString())) { 56 | transactionEvent.setType(requestEnvelop.getType()); 57 | kafkaTemplate.send(TOPIC, transactionEvent); 58 | return transactionEvent; 59 | } 60 | } 61 | throw new ServiceException(LedgerServiceErrorMessage.TYPE_NOT_SUPPORTED); 62 | } 63 | 64 | 65 | } 66 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # client-ledger-service: a REST endpoint for accepting and generating mutation events 2 | 3 | ## What does it do 4 | 5 | client-ledger-service is part 1 of the 3 parts of a simple ledger demo over event-sourcing, using commoditized software and simple set up 6 | 7 | ## Quick start 8 | 9 | ### Step 1 10 | 11 | Go to stress test folder 12 | ``` 13 | cd stress 14 | ``` 15 | 16 | Run Kafka (Assuming you are running on Mac, and you have already installed Apache Kafka). 17 | 18 | *The script uses default config which has only 1 partition. Make sure you config to at least the number of consumers (say 3).* 19 | ``` 20 | sudo ./kafka.sh 21 | ``` 22 | 23 | ### Step 2 24 | 25 | Run the End to End Stress Test, which send 1,000 p2p transactions into the http endpoint. 26 | ``` 27 | ./e2e.sh 28 | ``` 29 | 30 | ### Step 3 31 | 32 | Get the two competing consumers. 33 | Maria-DB based consumer: 34 | ``` 35 | git clone https://github.com/andrewkkchan/client-ledger-core-db.git 36 | ``` 37 | Redis based consumer: 38 | ``` 39 | git clone https://github.com/andrewkkchan/client-ledger-core-redis.git 40 | ``` 41 | 42 | ## Abstract 43 | 44 | Common practices of relying on ACID properties of database in resolving conflicts and recovering from failure would not scale in high-concurrency, low-latency use cases (e.g., trading, betting, payment). 45 | This sharing offers a quick overview of event sourcing framework, and how one can build a fully working ledger using simply commodity, open sourced products (e.g., Apache Kafka). 46 | This sharing further explores a more complicated use cases, where stochastic scenarios could be baked into traditional event sourcing to produce real time, big data type, operational intelligence. 47 | 48 | ## Why Event Sourcing? 49 | Ledger which relies on database ACID properties will handle up to 10-20 transaction per second. When pushed over that, there is a chance that the business rule (e.g., client ledger balance must not be overdrawn, a.k.a., <0) will be broken by racing condition and thread visibility. 50 | This code base is a basic set up for mobile/web environment to install event sourcing ledger functionality. And even with such a primitive set up, the http endpoint can easily handle 1,000 transactions per second, with 100% guarantee of holding business rules. 51 | 52 | ## What about 1 million per second? 53 | In commercial products with trading, betting and payment enterprises, I have achieved 1 million transaction per second while holding all business rules intact. A step-by-step customization and specialization on software and hardware is required. 54 | 55 | ### Step 1: To 10,000 per second 56 | * Replace HTTP endpoint with TCP endpoint. For example, use Kafka/Solace as incoming message handlers -- which would remove the limitation of HTTP protocol and HTTP server overhead. 57 | 58 | ### Step 2: To 100,000 per second 59 | * Replace Messaging endpoint on TCP with those on UDP. You will however need to maintain reliable in sequence transfer of events. 60 | * Replace non-positional data encoding (e.g., JSON, XML) for positional data encoding (e.g., protobuf, SBE) 61 | * Install competing consumers, with at least one of which being implemented in-memory, to cope with the burst of events and support query during burst. Persisting consumer will eventually catch up. 62 | 63 | ### Step 3: To 1,000,000 per second 64 | * Avoid garbage collection using pre-allocated ring buffer and other technique. 65 | 66 | ### Contact 67 | You can follow me on https://medium.com/@andrewchan_73514 68 | 69 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | org.springframework.boot 6 | spring-boot-starter-parent 7 | 2.1.2.RELEASE 8 | 9 | com.industrieit.ledger 10 | client-ledger-service 11 | jar 12 | client-ledger-service 13 | 1.0.0-SNAPSHOT 14 | 15 | 1.8 16 | ${java.version} 17 | ${java.version} 18 | 2.7.0 19 | 1.5.0 20 | 4.0.2 21 | 3.0.0 22 | 2.22.0 23 | 3.0.1 24 | 3.7.1 25 | 2.3 26 | 3.1.1 27 | 2.12.1 28 | 1.16 29 | 3.0.0 30 | 3.0.0 31 | 2.8.0 32 | 0.5.3 33 | 1.0.0 34 | 2.7.0 35 | 2.2.2.RELEASE 36 | 37 | 38 | 39 | 40 | 41 | 42 | org.springframework.boot 43 | spring-boot-maven-plugin 44 | 45 | 46 | 47 | repackage 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | org.springframework.boot 58 | spring-boot-starter-aop 59 | 60 | 61 | org.springframework.boot 62 | spring-boot-starter-logging 63 | 64 | 65 | org.springframework.boot 66 | spring-boot-starter-web 67 | 68 | 69 | org.springframework.boot 70 | spring-boot-starter-test 71 | test 72 | 73 | 74 | org.springframework.kafka 75 | spring-kafka 76 | 77 | 78 | org.springframework.boot 79 | spring-boot-autoconfigure 80 | 81 | 82 | 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /src/test/java/com/industrieit/ledger/clientledger/web/controller/TransactionControllerTest.java: -------------------------------------------------------------------------------- 1 | package com.industrieit.ledger.clientledger.web.controller; 2 | 3 | import com.fasterxml.jackson.core.io.JsonEOFException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.industrieit.ledger.clientledger.web.entity.TransactionEvent; 6 | import com.industrieit.ledger.clientledger.web.exception.LedgerServiceErrorMessage; 7 | import com.industrieit.ledger.clientledger.web.exception.ServiceException; 8 | import com.industrieit.ledger.clientledger.web.model.ledger.Type; 9 | import com.industrieit.ledger.clientledger.web.model.request.RequestEnvelop; 10 | import org.junit.Assert; 11 | import org.junit.Before; 12 | import org.junit.Rule; 13 | import org.junit.Test; 14 | import org.junit.rules.ExpectedException; 15 | import org.mockito.InjectMocks; 16 | import org.mockito.Mock; 17 | import org.mockito.Mockito; 18 | import org.mockito.MockitoAnnotations; 19 | import org.springframework.kafka.core.KafkaTemplate; 20 | 21 | import java.io.IOException; 22 | 23 | import static org.mockito.ArgumentMatchers.nullable; 24 | 25 | public class TransactionControllerTest { 26 | @Rule 27 | public ExpectedException thrown = ExpectedException.none(); 28 | 29 | @Mock 30 | private ObjectMapper objectMapper; 31 | @Mock 32 | private KafkaTemplate kafkaTemplate; 33 | @InjectMocks 34 | private TransactionController transactionController; 35 | 36 | @Before 37 | public void before() { 38 | MockitoAnnotations.initMocks(this); 39 | Mockito.when(kafkaTemplate.send(nullable(String.class), nullable(TransactionEvent.class))).thenReturn(null); 40 | } 41 | 42 | @Test 43 | public void testQueueTransaction() throws IOException { 44 | RequestEnvelop requestEnvelop = new RequestEnvelop(); 45 | requestEnvelop.setId("1234"); 46 | requestEnvelop.setType(Type.P2P.toString()); 47 | requestEnvelop.setRequest(null); 48 | Mockito.when(objectMapper.writeValueAsString(nullable(Object.class))).thenReturn("{}"); 49 | TransactionEvent transactionEvent = transactionController.queueTransaction(requestEnvelop); 50 | Assert.assertNotNull(transactionEvent); 51 | } 52 | 53 | @Test 54 | public void testQueueTransaction_typeNotSupported() throws IOException { 55 | thrown.expect(ServiceException.class); 56 | thrown.expectMessage(LedgerServiceErrorMessage.TYPE_NOT_SUPPORTED.getMessageKey()); 57 | RequestEnvelop requestEnvelop = new RequestEnvelop(); 58 | requestEnvelop.setId("1234"); 59 | requestEnvelop.setType("burn-it-down"); 60 | requestEnvelop.setRequest(null); 61 | Mockito.when(objectMapper.writeValueAsString(nullable(Object.class))).thenReturn("{}"); 62 | TransactionEvent transactionEvent = transactionController.queueTransaction(requestEnvelop); 63 | Assert.assertNull(transactionEvent); 64 | } 65 | 66 | @Test 67 | public void testQueueTransaction_requestUnreadable() throws IOException { 68 | thrown.expect(ServiceException.class); 69 | thrown.expectMessage(LedgerServiceErrorMessage.REQUEST_UNREADABLE.getMessageKey()); 70 | RequestEnvelop requestEnvelop = new RequestEnvelop(); 71 | requestEnvelop.setId("1234"); 72 | requestEnvelop.setType(Type.P2P.toString()); 73 | requestEnvelop.setRequest(null); 74 | Mockito.when(objectMapper.writeValueAsString(nullable(Object.class))).thenThrow(new JsonEOFException(null, null, null)); 75 | TransactionEvent transactionEvent = transactionController.queueTransaction(requestEnvelop); 76 | Assert.assertNull(transactionEvent); 77 | } 78 | 79 | @Test 80 | public void testQueueTransaction_idempotency() throws IOException { 81 | RequestEnvelop requestEnvelop = new RequestEnvelop(); 82 | requestEnvelop.setId("1234"); 83 | requestEnvelop.setType(Type.P2P.toString()); 84 | requestEnvelop.setRequest(null); 85 | Mockito.when(objectMapper.writeValueAsString(nullable(Object.class))).thenReturn("{}"); 86 | TransactionEvent transactionEvent = transactionController.queueTransaction(requestEnvelop); 87 | Assert.assertNotNull(transactionEvent); 88 | } 89 | } 90 | --------------------------------------------------------------------------------