├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.yml │ ├── config.yml │ └── feature-request.yml ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml ├── extract-snippets │ ├── README-SNIPPETS.txt │ ├── README.md │ ├── extract-snippets.py │ ├── extract-snippets.sh │ ├── snippet-extensions-more.yml │ └── snippet-extensions.yml ├── semantic.yml └── workflows │ ├── build-pull-request.yml │ ├── close-stale-issues.yml │ ├── closed-issue-message.yml │ ├── extract-snippets-dryrun.yml │ ├── extract-snippets.yml │ └── handle-stale-discussions.yml ├── .gitignore ├── .gitmodules ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── assets └── images │ └── hft-client-sequence-diagram.png ├── cpp-client ├── .gitignore ├── CMakeLists.txt ├── Config.cpp ├── Config.h ├── ExchangeClient.cpp ├── ExchangeClient.h ├── ExchangeClientLatencyTestHandler.cpp ├── ExchangeClientLatencyTestHandler.h ├── ExchangeProtocol.cpp ├── ExchangeProtocol.h ├── Logger.h ├── README.md ├── config.properties └── main.cpp ├── dependency-reduced-pom.xml ├── deployment ├── ansible │ ├── ansible.cfg │ ├── cleanup_all_clients.yaml │ ├── fetch_histogram_logs.yaml │ ├── inventory │ │ ├── client_inventory.aws_ec2.yml │ │ ├── inventory.aws_ec2.yml │ │ └── server_inventory.aws_ec2.yml │ ├── provision_ec2.yaml │ ├── restart_hft_client.yaml │ ├── restart_mock_trading_server.yaml │ ├── start_latency_test.yaml │ ├── stop_latency_test.yaml │ ├── tune_os.yaml │ └── validate_tuning_os.yaml ├── cdk │ ├── README.md │ ├── bin │ │ └── trading-benchmark.ts │ ├── cdk.json │ ├── jest.config.js │ ├── lib │ │ ├── cluster-placement-group-stack.ts │ │ ├── multi-az-stack.ts │ │ └── single-instance-stack.ts │ ├── package.json │ ├── template.yaml │ ├── test │ │ └── low-latency-connect.test.ts │ └── tsconfig.json ├── deploy.sh ├── latency_report.sh ├── output.csv ├── run_client.sh ├── run_ping_test.sh ├── run_server.sh ├── show_latency_reports.sh ├── stop.sh └── tune.sh ├── microbenchmarks └── src │ └── test │ └── java │ └── com │ └── aws │ └── trading │ ├── DeserializationBenchmark.java │ └── SerializationBenchmark.java ├── mock-trading-server ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── README.md ├── configuration.toml └── src │ ├── main.rs │ ├── websocket.rs │ └── websocket_message_types.rs ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── aws │ │ └── trading │ │ ├── Config.java │ │ ├── EndpointPingClient.java │ │ ├── ExchangeClient.java │ │ ├── ExchangeClientLatencyTestHandler.java │ │ ├── ExchangeProtocol.java │ │ ├── ExchangeProtocolImpl.java │ │ ├── LatencyReport.java │ │ ├── LatencyTools.java │ │ ├── Main.java │ │ ├── OutputLogLatencyReportGenerator.java │ │ ├── RoundTripLatencyTester.java │ │ └── StringMath.java └── resources │ ├── config.properties │ └── log4j2.xml └── test └── java └── ConfigTest.java /.github/ISSUE_TEMPLATE/bug-report.yml: -------------------------------------------------------------------------------- 1 | name: "🐛 Bug Report" 2 | description: Report a bug 3 | title: "(short issue description)" 4 | labels: [bug, needs-triage] 5 | assignees: [] 6 | body: 7 | - type: textarea 8 | id: description 9 | attributes: 10 | label: Describe the bug 11 | description: What is the problem? A clear and concise description of the bug. 12 | validations: 13 | required: true 14 | - type: textarea 15 | id: expected 16 | attributes: 17 | label: Expected Behavior 18 | description: | 19 | What did you expect to happen? 20 | validations: 21 | required: true 22 | - type: textarea 23 | id: current 24 | attributes: 25 | label: Current Behavior 26 | description: | 27 | What actually happened? 28 | 29 | Please include full errors, uncaught exceptions, stack traces, and relevant logs. 30 | If service responses are relevant, please include wire logs. 31 | validations: 32 | required: true 33 | - type: textarea 34 | id: reproduction 35 | attributes: 36 | label: Reproduction Steps 37 | description: | 38 | Provide a self-contained, concise snippet of code that can be used to reproduce the issue. 39 | For more complex issues provide a repo with the smallest sample that reproduces the bug. 40 | 41 | Avoid including business logic or unrelated code, it makes diagnosis more difficult. 42 | The code sample should be an SSCCE. See http://sscce.org/ for details. In short, please provide a code sample that we can copy/paste, run and reproduce. 43 | validations: 44 | required: true 45 | - type: textarea 46 | id: solution 47 | attributes: 48 | label: Possible Solution 49 | description: | 50 | Suggest a fix/reason for the bug 51 | validations: 52 | required: false 53 | - type: textarea 54 | id: context 55 | attributes: 56 | label: Additional Information/Context 57 | description: | 58 | Anything else that might be relevant for troubleshooting this bug. Providing context helps us come up with a solution that is most useful in the real world. 59 | validations: 60 | required: false 61 | 62 | - type: input 63 | id: cdk-version 64 | attributes: 65 | label: CDK CLI Version 66 | description: Output of `cdk version` 67 | validations: 68 | required: true 69 | 70 | - type: input 71 | id: framework-version 72 | attributes: 73 | label: Framework Version 74 | validations: 75 | required: false 76 | 77 | - type: input 78 | id: node-version 79 | attributes: 80 | label: Node.js Version 81 | validations: 82 | required: true 83 | 84 | - type: input 85 | id: operating-system 86 | attributes: 87 | label: OS 88 | validations: 89 | required: true 90 | 91 | - type: dropdown 92 | id: language 93 | attributes: 94 | label: Language 95 | multiple: true 96 | options: 97 | - Typescript 98 | - Python 99 | - .NET 100 | - Java 101 | - Go 102 | validations: 103 | required: true 104 | 105 | - type: input 106 | id: language-version 107 | attributes: 108 | label: Language Version 109 | description: E.g. TypeScript (3.8.3) | Java (8) | Python (3.7.3) 110 | validations: 111 | required: false 112 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | blank_issues_enabled: false 3 | contact_links: 4 | - name: 💬 General Question 5 | url: https://github.com/aws-samples/trading-latency-benchmark/discussions/categories/q-a 6 | about: Please ask and answer questions as a discussion thread 7 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🚀 Feature Request 3 | description: Suggest an idea for this project 4 | title: "(short issue description)" 5 | labels: [feature-request, needs-triage] 6 | assignees: [] 7 | body: 8 | - type: textarea 9 | id: description 10 | attributes: 11 | label: Describe the feature 12 | description: A clear and concise description of the feature you are proposing. 13 | validations: 14 | required: true 15 | - type: textarea 16 | id: use-case 17 | attributes: 18 | label: Use Case 19 | description: | 20 | Why do you need this feature? For example: "I'm always frustrated when..." 21 | validations: 22 | required: true 23 | - type: textarea 24 | id: solution 25 | attributes: 26 | label: Proposed Solution 27 | description: | 28 | Suggest how to implement the addition or change. Please include prototype/workaround/sketch/reference implementation. 29 | validations: 30 | required: false 31 | - type: textarea 32 | id: other 33 | attributes: 34 | label: Other Information 35 | description: | 36 | Any alternative solutions or features you considered, a more detailed explanation, stack traces, related issues, links for context, etc. 37 | validations: 38 | required: false 39 | - type: checkboxes 40 | id: ack 41 | attributes: 42 | label: Acknowledgements 43 | options: 44 | - label: I may be able to implement this feature request 45 | required: false 46 | - label: This feature might incur a breaking change 47 | required: false 48 | 49 | - type: dropdown 50 | id: language 51 | attributes: 52 | label: Language 53 | multiple: true 54 | options: 55 | - Typescript 56 | - Python 57 | - .NET 58 | - Java 59 | - Go 60 | - Rust 61 | - Shell 62 | validations: 63 | required: true 64 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 9 | 10 | Fixes # 11 | 12 | --- 13 | 14 | By submitting this pull request, I confirm that my contribution is made under the terms of the [MIT-0 License]. 15 | 16 | [MIT-0 License]: https://github.com/aws/mit-0 17 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "maven" 4 | directory: "/java" 5 | schedule: 6 | interval: "daily" 7 | open-pull-requests-limit: 10 8 | - package-ecosystem: "github-actions" 9 | directory: "/" 10 | schedule: 11 | interval: "weekly" 12 | open-pull-requests-limit: 10 13 | -------------------------------------------------------------------------------- /.github/extract-snippets/README-SNIPPETS.txt: -------------------------------------------------------------------------------- 1 | The snippet files in this directory are extracted from source files elsewhere 2 | in this repository by an automated process. Do not make changes to snippets 3 | yourself; your changes will likely be lost in the not-so-distant future. -------------------------------------------------------------------------------- /.github/extract-snippets/extract-snippets.sh: -------------------------------------------------------------------------------- 1 | # extract-snippets.sh v1.1.0 2 | # Jerry Kindall, Amazon Web Services 3 | 4 | # this script manages updating of snippets found in source code to the snippets branch 5 | 6 | # the args to this script constitute the command that is executed to get the list of source 7 | # files to be processed. for example, if it is run on push, pass e.g.: git diff @^ --name-only 8 | # to process only the files included in the last push. to process all files, pass e.g.: 9 | # find . --type f 10 | 11 | # snippets are never deleted, as this would break any document where they're used. 12 | 13 | mkdir ../snippets 14 | cp .github/extract-snippets/README-SNIPPETS.txt ../snippets/README.txt 15 | 16 | python -m pip install --upgrade pip pyyaml 17 | find . -type f | python .github/extract-snippets/extract-snippets.py ../snippets || exit 1 18 | 19 | if [[ -z $1 ]]; then 20 | 21 | git checkout --track origin/snippets 22 | mkdir -p snippets 23 | cp ../snippets/* snippets 24 | 25 | git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" 26 | git config --local user.name "github-actions[bot]" 27 | git add --all 28 | git commit -m "Updated on `date -R`" 29 | 30 | fi 31 | 32 | exit 0 -------------------------------------------------------------------------------- /.github/extract-snippets/snippet-extensions-more.yml: -------------------------------------------------------------------------------- 1 | # This file maps filename extensions (or, really, any string ending a path, 2 | # such as /makefile) to comemnt markers for a particular language, so that 3 | # snippet directives can be detected in that source file 4 | 5 | somewhere/lambda/widgets.js: "" # example of excluding a file via empty comment marker 6 | 7 | .ahk: ; # AutoHotkey 8 | .bash: "#" # bash shell (must quote "#" here as it is also the YAML comment marker) 9 | .bat: "REM @REM rem @rem Rem @Rem" # Windows batch file 10 | .class: // # Java 11 | .c: "/*" # C (closing comment marker must be on same line) 12 | .cpp: // # C++ 13 | .csh: "#" # C shell 14 | .cs: // # C# 15 | .dart: // # Dart 16 | .f: "* c !" # FORTRAN (fixed field; most compilers also support ! even in fexed field) 17 | .f90: "!" # FORTRAN (freeform) 18 | .fs: // # F# 19 | .h: "/* //" # C or Objective-C header (closing comment marker must be on same line) 20 | .hpp: // # C++ header 21 | .go: // # Go 22 | .html: 32 | 33 | 34 | 35 | com.fasterxml.jackson.core 36 | jackson-databind 37 | 2.14.2 38 | 39 | 40 | io.netty 41 | netty-all 42 | 4.1.94.Final 43 | 44 | 45 | net.openhft 46 | chronicle-wire 47 | 2.24ea23 48 | 49 | 50 | com.alibaba 51 | fastjson 52 | 2.0.31 53 | 54 | 55 | com.lmax 56 | disruptor 57 | 4.0.0.RC1 58 | 59 | 60 | 61 | net.openhft 62 | chronicle-logger 63 | 4.21.82 64 | pom 65 | 66 | 67 | org.openjdk.jmh 68 | jmh-core 69 | 1.35 70 | 71 | 72 | org.openjdk.jmh 73 | jmh-generator-annprocess 74 | 1.35 75 | 76 | 77 | io.netty.incubator 78 | netty-incubator-transport-native-io_uring 79 | 0.0.21.Final 80 | linux-x86_64 81 | 82 | 83 | io.netty 84 | netty-transport-native-epoll 85 | 4.1.94.Final 86 | linux-x86_64 87 | 88 | 89 | io.netty 90 | netty-tcnative-boringssl-static 91 | 2.0.64.Final 92 | ${os.detected.classifier} 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | kr.motd.maven 101 | os-maven-plugin 102 | 1.7.1 103 | 104 | 105 | 106 | 107 | org.apache.maven.plugins 108 | maven-surefire-plugin 109 | 3.0.0-M7 110 | 111 | 112 | UTC 113 | 114 | 1 115 | true 116 | hourly 117 | 118 | 119 | 120 | org.apache.maven.plugins 121 | maven-compiler-plugin 122 | 123 | 124 | -Xlint:deprecation 125 | -parameters 126 | 127 | 11 128 | 11 129 | UTF-8 130 | 131 | 132 | 133 | org.apache.maven.plugins 134 | maven-source-plugin 135 | 3.2.1 136 | 137 | 138 | attach-sources 139 | 140 | jar 141 | test-jar 142 | 143 | 144 | 145 | 146 | 147 | 148 | org.apache.maven.plugins 149 | maven-jar-plugin 150 | 3.1.0 151 | 152 | 153 | 154 | true 155 | one.example.Main 156 | 157 | 158 | 159 | 160 | 161 | org.apache.maven.plugins 162 | maven-shade-plugin 163 | 3.2.3 164 | 165 | 166 | package 167 | 168 | shade 169 | 170 | 171 | 172 | 174 | com.aws.trading.Main 175 | 176 | 177 | 178 | 179 | *:* 180 | 181 | META-INF/*.SF 182 | META-INF/*.DSA 183 | META-INF/*.RSA 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | -------------------------------------------------------------------------------- /src/main/java/com/aws/trading/EndpointPingClient.java: -------------------------------------------------------------------------------- 1 | package com.aws.trading; 2 | 3 | import org.HdrHistogram.Histogram; 4 | import org.HdrHistogram.SingleWriterRecorder; 5 | import org.apache.logging.log4j.LogManager; 6 | import org.apache.logging.log4j.Logger; 7 | 8 | import java.io.File; 9 | import java.io.IOException; 10 | import java.io.PrintStream; 11 | import java.net.InetAddress; 12 | import java.net.InetSocketAddress; 13 | import java.net.Socket; 14 | import java.util.LinkedHashMap; 15 | import java.util.Timer; 16 | import java.util.TimerTask; 17 | 18 | import static com.aws.trading.Config.*; 19 | import static com.aws.trading.LatencyTools.formatNanos; 20 | import static com.aws.trading.RoundTripLatencyTester.getLogFile; 21 | import static com.aws.trading.RoundTripLatencyTester.saveHistogramToFile; 22 | 23 | public class EndpointPingClient { 24 | private static final Logger LOGGER = LogManager.getLogger(EndpointPingClient.class); 25 | 26 | public static void main(String[] args) throws IOException { 27 | Timer timer = new Timer(); 28 | String[] hosts = HOST.split(","); 29 | 30 | for (String host : hosts) { 31 | var arr = host.trim().split(":"); 32 | timer.scheduleAtFixedRate(arr.length > 1 ? new PingWithHostAndPort(arr[0], Integer.parseInt(arr[1])) : new PingWithHost(arr[0]), 0, PING_INTERVAL); 33 | } 34 | } 35 | 36 | private abstract static class PingTask extends TimerTask { 37 | final Histogram HISTOGRAM = new Histogram(Long.MAX_VALUE, 2); 38 | protected final String host; 39 | private final PrintStream histogramLogFile; 40 | private long count = 0; 41 | private final SingleWriterRecorder hdrRecorder = new SingleWriterRecorder(Long.MAX_VALUE, 2); 42 | 43 | public PingTask(String host) throws IOException { 44 | this.host = host; 45 | String folderPath = "./" + host.replace(".", "_"); 46 | File folder = new File(folderPath); 47 | if (!folder.exists()) { 48 | folder.mkdirs(); // Create the folder if it doesn't exist 49 | } 50 | this.histogramLogFile = getLogFile(folderPath + "/histogram.hlog"); 51 | } 52 | 53 | abstract long measureRTT() throws IOException; 54 | 55 | @Override 56 | public void run() { 57 | try { 58 | long latency = measureRTT(); 59 | count += 1; 60 | if (count < WARMUP_COUNT) { 61 | LOGGER.info("warming up... - message count: {}", count); 62 | return; 63 | } 64 | 65 | LOGGER.info(" {} is reachable. Latency: {}", host, formatNanos(latency)); 66 | hdrRecorder.recordValue(latency); 67 | if (count % TEST_SIZE == 0) { 68 | HISTOGRAM.add(hdrRecorder.getIntervalHistogram()); 69 | LinkedHashMap latencyReport = LatencyTools.createLatencyReport(HISTOGRAM); 70 | saveHistogramToFile(HISTOGRAM, System.nanoTime(), histogramLogFile); 71 | LOGGER.info("Percentiles for host: {}\n {}\n", host, LatencyTools.toJSON(latencyReport)); 72 | } 73 | } catch (IOException e) { 74 | LOGGER.error("{} is not reachable.", host); 75 | throw new RuntimeException(e); 76 | } 77 | } 78 | } 79 | 80 | private static class PingWithHost extends PingTask { 81 | public PingWithHost(String host) throws IOException { 82 | super(host); 83 | } 84 | @Override 85 | long measureRTT() throws IOException { 86 | InetAddress address = InetAddress.getByName(host); 87 | long startTime = System.nanoTime(); 88 | if(address.isReachable(5000)){ 89 | long endTime = System.nanoTime(); 90 | return endTime - startTime; 91 | } else { 92 | throw new IOException("address is not reachable"); 93 | } 94 | } 95 | } 96 | 97 | private static class PingWithHostAndPort extends PingTask { 98 | private final Integer port; 99 | 100 | public PingWithHostAndPort(String host, Integer port) throws IOException { 101 | super(host); 102 | this.port = port; 103 | } 104 | 105 | @Override 106 | long measureRTT() throws IOException { 107 | InetSocketAddress socketAddress = new InetSocketAddress(host, port); 108 | Socket socket = new Socket(); 109 | long startTime = System.nanoTime(); 110 | socket.connect(socketAddress, 5000); // Timeout after 5 seconds 111 | long endTime = System.nanoTime(); 112 | socket.close(); 113 | return endTime - startTime; 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/main/java/com/aws/trading/ExchangeProtocol.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * SPDX-License-Identifier: MIT-0 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 6 | * software and associated documentation files (the "Software"), to deal in the Software 7 | * without restriction, including without limitation the rights to use, copy, modify, 8 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 12 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 13 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 14 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 16 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | */ 18 | package com.aws.trading; 19 | 20 | import io.netty.buffer.ByteBuf; 21 | import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; 22 | 23 | public interface ExchangeProtocol { 24 | TextWebSocketFrame createBuyOrder(String pair, String clientId); 25 | 26 | ByteBuf createSellOrder(String pair, String clientId); 27 | 28 | ByteBuf createOrder(String pair, String type, String uuid, String side, String price, String qty); 29 | 30 | TextWebSocketFrame createCancelOrder(String pair, String clientid); 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/aws/trading/ExchangeProtocolImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * SPDX-License-Identifier: MIT-0 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 6 | * software and associated documentation files (the "Software"), to deal in the Software 7 | * without restriction, including without limitation the rights to use, copy, modify, 8 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 12 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 13 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 14 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 16 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | */ 18 | package com.aws.trading; 19 | 20 | import io.netty.buffer.ByteBuf; 21 | import io.netty.buffer.Unpooled; 22 | import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; 23 | import io.netty.util.CharsetUtil; 24 | import io.netty.util.internal.PlatformDependent; 25 | 26 | import java.nio.charset.StandardCharsets; 27 | import java.util.Base64; 28 | 29 | public class ExchangeProtocolImpl implements ExchangeProtocol { 30 | public static final byte[] AUTH_MSG_HEADER = "{\"type\":\"AUTHENTICATE\",\"api_token\":\"".getBytes(StandardCharsets.UTF_8); 31 | final static byte[] HEADER = "{\"type\":\"CREATE_ORDER\",\"order\":{\"instrument_code\":\"".getBytes(StandardCharsets.UTF_8); 32 | final static byte[] SYMBOL_END = "\",\"client_id\":\"".getBytes(StandardCharsets.UTF_8); 33 | final static byte[] CLIENT_ID_END = "\",\"side\":\"".getBytes(StandardCharsets.UTF_8); 34 | final static byte[] buySide = "BUY".getBytes(StandardCharsets.UTF_8); 35 | final static byte[] sellSide = "SELL".getBytes(StandardCharsets.UTF_8); 36 | final static byte[] SIDE_END = "\",\"type\":\"".getBytes(StandardCharsets.UTF_8); 37 | final static byte[] dummyType = "LIMIT".getBytes(StandardCharsets.UTF_8); 38 | final static byte[] TYPE_END = "\",\"price\":\"".getBytes(StandardCharsets.UTF_8); 39 | final static byte[] dummyBuyPrice = "1".getBytes(StandardCharsets.UTF_8); 40 | final static byte[] dummySellPrice = "2".getBytes(StandardCharsets.UTF_8); 41 | final static byte[] PRICE_END = "\",\"amount\":\"".getBytes(StandardCharsets.UTF_8); 42 | final static byte[] dummyAmount = "1".getBytes(StandardCharsets.UTF_8); 43 | final static byte[] AMOUNT_END = "\",\"time_in_force\":\"".getBytes(StandardCharsets.UTF_8); 44 | final static byte[] dummyTimeInForce = "GOOD_TILL_CANCELLED".getBytes(StandardCharsets.UTF_8); 45 | final static byte[] TIME_IN_FORCE_END = "\"}}".getBytes(StandardCharsets.UTF_8); 46 | final static byte[] CANCEL_ORDER_HEADER = "{\"type\":\"CANCEL_ORDER\",\"client_id\":\"".getBytes(StandardCharsets.UTF_8); 47 | final static byte[] CANCEL_ORDER_CLIENT_ID_END = "\",\"instrument_code\":\"".getBytes(StandardCharsets.UTF_8); 48 | final static byte[] MSG_END = "\"}".getBytes(StandardCharsets.UTF_8); 49 | final static byte[] SUBSCRIBE_MSG = "{\"type\":\"SUBSCRIBE\",\"channels\":[{\"name\":\"ORDERS\"}]}".getBytes(StandardCharsets.UTF_8); 50 | 51 | static byte[] randomBytes(int size) { 52 | byte[] bytes = new byte[size]; 53 | PlatformDependent.threadLocalRandom().nextBytes(bytes); 54 | return bytes; 55 | } 56 | static String base64(byte[] data) { 57 | if (PlatformDependent.javaVersion() >= 8) { 58 | return Base64.getEncoder().encodeToString(data); 59 | } else { 60 | ByteBuf encodedData = Unpooled.wrappedBuffer(data); 61 | 62 | String encodedString; 63 | try { 64 | ByteBuf encoded = io.netty.handler.codec.base64.Base64.encode(encodedData); 65 | 66 | try { 67 | encodedString = encoded.toString(CharsetUtil.UTF_8); 68 | } finally { 69 | encoded.release(); 70 | } 71 | } finally { 72 | encodedData.release(); 73 | } 74 | 75 | return encodedString; 76 | } 77 | } 78 | 79 | public TextWebSocketFrame createBuyOrder(String pair, String clientId) { 80 | return new TextWebSocketFrame(Unpooled.wrappedBuffer( 81 | ExchangeProtocolImpl.HEADER, 82 | pair.getBytes(StandardCharsets.UTF_8), ExchangeProtocolImpl.SYMBOL_END, 83 | clientId.getBytes(StandardCharsets.UTF_8), ExchangeProtocolImpl.CLIENT_ID_END, 84 | ExchangeProtocolImpl.buySide, ExchangeProtocolImpl.SIDE_END, 85 | ExchangeProtocolImpl.dummyType, ExchangeProtocolImpl.TYPE_END, 86 | ExchangeProtocolImpl.dummyBuyPrice, ExchangeProtocolImpl.PRICE_END, 87 | ExchangeProtocolImpl.dummyAmount, ExchangeProtocolImpl.AMOUNT_END, 88 | ExchangeProtocolImpl.dummyTimeInForce, ExchangeProtocolImpl.TIME_IN_FORCE_END 89 | )); 90 | } 91 | 92 | public ByteBuf createSellOrder(String pair, String clientId) { 93 | return Unpooled.wrappedBuffer( 94 | ExchangeProtocolImpl.HEADER, 95 | pair.getBytes(StandardCharsets.UTF_8), ExchangeProtocolImpl.SYMBOL_END, 96 | clientId.getBytes(StandardCharsets.UTF_8), ExchangeProtocolImpl.CLIENT_ID_END, 97 | ExchangeProtocolImpl.sellSide, ExchangeProtocolImpl.SIDE_END, 98 | ExchangeProtocolImpl.dummyType, ExchangeProtocolImpl.TYPE_END, 99 | ExchangeProtocolImpl.dummySellPrice, ExchangeProtocolImpl.PRICE_END, 100 | ExchangeProtocolImpl.dummyAmount, ExchangeProtocolImpl.AMOUNT_END, 101 | ExchangeProtocolImpl.dummyTimeInForce, ExchangeProtocolImpl.TIME_IN_FORCE_END 102 | ); 103 | } 104 | 105 | public ByteBuf createOrder(String pair, String type, String uuid, String side, String price, String qty) { 106 | return Unpooled.wrappedBuffer( 107 | ExchangeProtocolImpl.HEADER, 108 | pair.getBytes(StandardCharsets.UTF_8), ExchangeProtocolImpl.SYMBOL_END, 109 | uuid.getBytes(StandardCharsets.UTF_8), ExchangeProtocolImpl.CLIENT_ID_END, 110 | side.getBytes(StandardCharsets.UTF_8), ExchangeProtocolImpl.SIDE_END, 111 | type.getBytes(StandardCharsets.UTF_8), ExchangeProtocolImpl.TYPE_END, 112 | price.getBytes(StandardCharsets.UTF_8), ExchangeProtocolImpl.PRICE_END, 113 | qty.getBytes(StandardCharsets.UTF_8), ExchangeProtocolImpl.AMOUNT_END, 114 | ExchangeProtocolImpl.dummyTimeInForce, ExchangeProtocolImpl.TIME_IN_FORCE_END 115 | ); 116 | } 117 | 118 | public TextWebSocketFrame createCancelOrder(String pair, String clientid) { 119 | return new TextWebSocketFrame(Unpooled.wrappedBuffer( 120 | ExchangeProtocolImpl.CANCEL_ORDER_HEADER, 121 | clientid.getBytes(StandardCharsets.UTF_8), 122 | ExchangeProtocolImpl.CANCEL_ORDER_CLIENT_ID_END, 123 | pair.getBytes(StandardCharsets.UTF_8), 124 | ExchangeProtocolImpl.MSG_END 125 | )); 126 | } 127 | 128 | } 129 | -------------------------------------------------------------------------------- /src/main/java/com/aws/trading/LatencyReport.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * SPDX-License-Identifier: MIT-0 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 6 | * software and associated documentation files (the "Software"), to deal in the Software 7 | * without restriction, including without limitation the rights to use, copy, modify, 8 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 12 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 13 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 14 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 16 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | */ 18 | package com.aws.trading; 19 | 20 | import org.HdrHistogram.Histogram; 21 | import org.HdrHistogram.HistogramLogReader; 22 | import org.apache.logging.log4j.LogManager; 23 | import org.apache.logging.log4j.Logger; 24 | 25 | import java.io.FileWriter; 26 | import java.io.IOException; 27 | import java.nio.file.Files; 28 | import java.nio.file.Path; 29 | import java.nio.file.Paths; 30 | import java.util.ArrayList; 31 | import java.util.LinkedHashMap; 32 | import java.util.List; 33 | import java.util.Map; 34 | import java.util.regex.Matcher; 35 | import java.util.regex.Pattern; 36 | import java.util.stream.Collectors; 37 | import java.util.stream.Stream; 38 | 39 | import static com.aws.trading.Main.printHelpMessage; 40 | import static java.nio.file.Files.newInputStream; 41 | 42 | public class LatencyReport { 43 | private static final Logger LOGGER = LogManager.getLogger(LatencyReport.class); 44 | private static final String regex = ".*/LowLatencyConnectStack/([^/]+)/home/ec2-user/([^/]+)/histogram.hlog"; 45 | private static final Pattern pattern = Pattern.compile(regex); 46 | 47 | public static void main(String[] args) { 48 | if (args.length < 2) { 49 | printHelpMessage(); 50 | System.exit(0); 51 | } 52 | 53 | // Check if we're processing a single file or a directory 54 | Path path = Paths.get(args[1]); 55 | if (Files.isRegularFile(path)) { 56 | // Process a single histogram file 57 | processSingleHistogramFile(path, args); 58 | } else { 59 | // Process a directory of histogram files 60 | try (Stream paths = Files.walk(path)) { 61 | List> rows = paths.filter(Files::isRegularFile) 62 | .filter(file -> file.toString().endsWith(".hlog")) 63 | .map(file -> { 64 | LOGGER.info("loading histogram from file {}", file); 65 | HistogramLogReader logReader = getHistogramLogReader(file); 66 | Histogram histogram = null; 67 | while (logReader.hasNext()) { 68 | Histogram iter = (Histogram) logReader.nextIntervalHistogram(); 69 | if (null == histogram) { 70 | histogram = iter; 71 | } else { 72 | histogram.add(iter); 73 | } 74 | } 75 | assert histogram != null; 76 | final LinkedHashMap row = LatencyTools.createLatencyReport(histogram); 77 | final Matcher matcher = pattern.matcher(file.toString()); 78 | if (matcher.matches()) { 79 | row.put("source", matcher.group(1)); 80 | row.put("destination", matcher.group(2)); 81 | } else { 82 | LOGGER.error("Unable to parse the input string: {}", file.toString()); 83 | } 84 | return row; 85 | }).collect(Collectors.toList()); 86 | 87 | // Write results to CSV 88 | if (!rows.isEmpty()) { 89 | String outputFile = args.length > 2 ? args[2] : "output.csv"; 90 | writeToCSV(rows, outputFile); 91 | System.out.println("CSV report written to: " + outputFile); 92 | } else { 93 | System.err.println("No histogram files found in directory: " + path); 94 | } 95 | } catch (IOException e) { 96 | System.err.println("Error processing histogram log files: " + e.getMessage()); 97 | } 98 | } 99 | } 100 | 101 | private static void processSingleHistogramFile(Path file, String[] args) { 102 | try { 103 | LOGGER.info("Processing histogram file: {}", file); 104 | HistogramLogReader logReader = getHistogramLogReader(file); 105 | Histogram histogram = null; 106 | 107 | while (logReader.hasNext()) { 108 | Histogram iter = (Histogram) logReader.nextIntervalHistogram(); 109 | if (null == histogram) { 110 | histogram = iter; 111 | } else { 112 | histogram.add(iter); 113 | } 114 | } 115 | 116 | if (histogram != null) { 117 | // Print formatted report to stdout for shell script to parse 118 | printFormattedReport(histogram); 119 | 120 | // Optionally save CSV if a filename is provided 121 | if (args.length > 2) { 122 | LinkedHashMap row = LatencyTools.createLatencyReport(histogram); 123 | List> rows = new ArrayList<>(); 124 | rows.add(row); 125 | writeToCSV(rows, args[2]); 126 | System.out.println("CSV report written to: " + args[2]); 127 | } 128 | } else { 129 | System.err.println("No histogram data found in file: " + file); 130 | } 131 | } catch (Exception e) { 132 | System.err.println("Error processing histogram file: " + e.getMessage()); 133 | } 134 | } 135 | 136 | private static void printFormattedReport(Histogram histogram) { 137 | // Print summary statistics in a format that can be easily parsed by the shell script 138 | System.out.println("Latency Report Summary"); 139 | System.out.println("====================="); 140 | System.out.println("Min latency: " + LatencyTools.formatNanos(histogram.getMinValue())); 141 | System.out.println("Max latency: " + LatencyTools.formatNanos(histogram.getMaxValue())); 142 | System.out.println("Mean latency: " + LatencyTools.formatNanos((long)histogram.getMean())); 143 | System.out.println("Total count: " + histogram.getTotalCount()); 144 | System.out.println(); 145 | 146 | System.out.println("Percentile Distribution"); 147 | System.out.println("======================"); 148 | for (double percentile : LatencyTools.PERCENTILES) { 149 | System.out.printf("%.2f%%: %s\n", 150 | percentile, 151 | LatencyTools.formatNanos(histogram.getValueAtPercentile(percentile))); 152 | } 153 | } 154 | 155 | public static void writeToCSV(List> data, String fileName) { 156 | try (FileWriter writer = new FileWriter(fileName)) { 157 | // Get the header columns from the first row 158 | List headers = new ArrayList<>(data.get(0).keySet()); 159 | 160 | // Write the header row 161 | writer.write(String.join(",", headers)); 162 | writer.write("\n"); 163 | 164 | // Write the data rows 165 | for (Map row : data) { 166 | List values = new ArrayList<>(); 167 | for (String header : headers) { 168 | values.add(row.get(header)); 169 | } 170 | writer.write(String.join(",", values)); 171 | writer.write("\n"); 172 | } 173 | } catch (IOException e) { 174 | LOGGER.error("Error writing to CSV file: {}", e.getMessage()); 175 | } 176 | } 177 | 178 | public static HistogramLogReader getHistogramLogReader(Path file) throws RuntimeException { 179 | try { 180 | return new HistogramLogReader(newInputStream(file)); 181 | } catch (IOException e) { 182 | throw new RuntimeException(e); 183 | } 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /src/main/java/com/aws/trading/LatencyTools.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Maksim Zheravin 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.aws.trading; 17 | 18 | import com.alibaba.fastjson2.JSON; 19 | import com.alibaba.fastjson2.JSONWriter; 20 | import org.HdrHistogram.Histogram; 21 | 22 | import java.util.Arrays; 23 | import java.util.LinkedHashMap; 24 | 25 | public final class LatencyTools { 26 | 27 | public static final double[] PERCENTILES = new double[]{50, 90, 95, 99, 99.9, 99.99}; 28 | 29 | public static String createLatencyReportJson(Histogram histogram) { 30 | return toJSON(createLatencyReport(histogram)); 31 | } 32 | 33 | public static LinkedHashMap createLatencyReport(Histogram histogram) { 34 | final LinkedHashMap fmt = new LinkedHashMap<>(); 35 | Arrays.stream(PERCENTILES).forEach(p -> 36 | fmt.put(p + "%", formatNanos(histogram.getValueAtPercentile(p))) 37 | ); 38 | fmt.put("W", formatNanos(histogram.getMaxValue())); 39 | return fmt; 40 | } 41 | 42 | public static String toJSON(LinkedHashMap fmt){ 43 | return JSON.toJSONString(fmt, JSONWriter.Feature.PrettyFormat); 44 | } 45 | 46 | public static String toCSV(LinkedHashMap fmt){ 47 | StringBuilder sb = new StringBuilder(); 48 | String header = String.join(",", fmt.keySet()); 49 | String values = String.join(",", fmt.values()); 50 | return sb.append(header).append("\n").append(values).toString(); 51 | } 52 | 53 | public static String formatNanos(long ns) { 54 | float value = ns / 1000f; 55 | String timeUnit = "µs"; 56 | if (value > 1000) { 57 | value /= 1000; 58 | timeUnit = "ms"; 59 | } 60 | 61 | if (value > 1000) { 62 | value /= 1000; 63 | timeUnit = "s"; 64 | } 65 | 66 | if (value < 3) { 67 | return Math.round(value * 100) / 100f + timeUnit; 68 | } else if (value < 30) { 69 | return Math.round(value * 10) / 10f + timeUnit; 70 | } else { 71 | return Math.round(value) + timeUnit; 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/aws/trading/Main.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * SPDX-License-Identifier: MIT-0 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 6 | * software and associated documentation files (the "Software"), to deal in the Software 7 | * without restriction, including without limitation the rights to use, copy, modify, 8 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 12 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 13 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 14 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 16 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | */ 18 | package com.aws.trading; 19 | 20 | import org.apache.logging.log4j.LogManager; 21 | import org.apache.logging.log4j.Logger; 22 | 23 | import java.io.IOException; 24 | import java.net.URISyntaxException; 25 | import java.security.KeyManagementException; 26 | import java.security.KeyStoreException; 27 | import java.security.NoSuchAlgorithmException; 28 | import java.security.UnrecoverableKeyException; 29 | import java.security.cert.CertificateException; 30 | 31 | /** 32 | * Main entry point for the trading latency benchmark application. 33 | * Provides command-line interface to run different types of tests and reports. 34 | */ 35 | public class Main { 36 | private static final Logger LOGGER = LogManager.getLogger(Main.class); 37 | 38 | /** 39 | * Main entry point for the application. 40 | * 41 | * @param args Command line arguments 42 | * @throws Exception If an error occurs during execution 43 | */ 44 | public static void main(String[] args) { 45 | try { 46 | if (args.length < 1) { 47 | printHelpMessage(); 48 | System.exit(0); 49 | } 50 | 51 | String command = args[0].toLowerCase(); 52 | executeCommand(command, args); 53 | } catch (Exception e) { 54 | LOGGER.error("Error executing command", e); 55 | System.err.println("Error: " + e.getMessage()); 56 | printHelpMessage(); 57 | System.exit(1); 58 | } 59 | } 60 | 61 | /** 62 | * Executes the specified command with the given arguments. 63 | * 64 | * @param command The command to execute 65 | * @param args The command line arguments 66 | * @throws URISyntaxException If there is an error with the URI syntax 67 | * @throws InterruptedException If the execution is interrupted 68 | * @throws IOException If an I/O error occurs 69 | * @throws UnrecoverableKeyException If the key cannot be recovered 70 | * @throws CertificateException If there is an error with the certificate 71 | * @throws NoSuchAlgorithmException If a required cryptographic algorithm is not available 72 | * @throws KeyStoreException If there is an error accessing the keystore 73 | * @throws KeyManagementException If there is an error with key management 74 | */ 75 | private static void executeCommand(String command, String[] args) 76 | throws URISyntaxException, InterruptedException, IOException, 77 | UnrecoverableKeyException, CertificateException, 78 | NoSuchAlgorithmException, KeyStoreException, KeyManagementException { 79 | 80 | switch (command) { 81 | case "latency-test": 82 | LOGGER.info("Starting latency test"); 83 | RoundTripLatencyTester.main(args); 84 | break; 85 | 86 | case "ping-latency": 87 | LOGGER.info("Starting ping latency test"); 88 | EndpointPingClient.main(args); 89 | break; 90 | 91 | case "latency-report": 92 | LOGGER.info("Generating latency report"); 93 | if (args.length < 2) { 94 | LOGGER.error("Missing report file path"); 95 | System.err.println("Error: Missing report file path"); 96 | printHelpMessage(); 97 | System.exit(1); 98 | } 99 | LatencyReport.main(args); 100 | break; 101 | 102 | case "help": 103 | printHelpMessage(); 104 | break; 105 | 106 | default: 107 | LOGGER.error("Unknown command: {}", command); 108 | System.err.println("Unknown command: " + command); 109 | printHelpMessage(); 110 | System.exit(1); 111 | } 112 | } 113 | 114 | /** 115 | * Prints the help message with usage instructions. 116 | */ 117 | public static void printHelpMessage() { 118 | System.out.println("Trading Latency Benchmark Tool"); 119 | System.out.println("=============================="); 120 | System.out.println("Usage: java -jar []"); 121 | System.out.println("\nCommands:"); 122 | System.out.println(" latency-test Run round-trip latency test between client and server"); 123 | System.out.println(" ping-latency Run ping latency test to measure network round-trip time"); 124 | System.out.println(" latency-report Generate and print latency report from log file"); 125 | System.out.println(" help Print this help message"); 126 | System.out.println("\nArguments for latency-report:"); 127 | System.out.println(" Path to the latency report file (.hlog)"); 128 | System.out.println("\nExamples:"); 129 | System.out.println(" java -jar ExchangeFlow-1.0-SNAPSHOT.jar latency-test"); 130 | System.out.println(" java -jar ExchangeFlow-1.0-SNAPSHOT.jar latency-report ./histogram_logs/latency.hlog"); 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /src/main/java/com/aws/trading/OutputLogLatencyReportGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * SPDX-License-Identifier: MIT-0 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 6 | * software and associated documentation files (the "Software"), to deal in the Software 7 | * without restriction, including without limitation the rights to use, copy, modify, 8 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 12 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 13 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 14 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 16 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | */ 18 | package com.aws.trading; 19 | 20 | import org.HdrHistogram.HistogramLogReader; 21 | import org.HdrHistogram.SingleWriterRecorder; 22 | import org.apache.logging.log4j.LogManager; 23 | import org.apache.logging.log4j.Logger; 24 | 25 | import java.io.IOException; 26 | import java.nio.file.Files; 27 | import java.nio.file.Path; 28 | import java.nio.file.Paths; 29 | import java.util.HashMap; 30 | import java.util.LinkedHashMap; 31 | import java.util.Map; 32 | import java.util.Objects; 33 | import java.util.regex.Matcher; 34 | import java.util.regex.Pattern; 35 | import java.util.stream.Stream; 36 | 37 | import static com.aws.trading.Main.printHelpMessage; 38 | import static java.nio.file.Files.newInputStream; 39 | import static java.util.stream.Collectors.toList; 40 | 41 | public class OutputLogLatencyReportGenerator { 42 | private static final Logger LOGGER = LogManager.getLogger(OutputLogLatencyReportGenerator.class); 43 | private static final Pattern LOG_PATTERN = Pattern.compile( 44 | ".*?(\\d+\\.\\d+\\.\\d+\\.\\d+)\\s*.*?\\s*Latency:\\s*(\\d+(\\.\\d+)?)(ms|µs).*", 45 | Pattern.CASE_INSENSITIVE 46 | ); 47 | private static final Pattern pattern = Pattern.compile(".*/LowLatencyConnectStack/([^/]+)/home/ec2-user/output.log"); 48 | 49 | public static void main(String[] args) { 50 | if (args.length < 2) { 51 | printHelpMessage(); 52 | System.exit(0); 53 | } 54 | try (Stream paths = Files.walk(Paths.get(args[1]))) { 55 | LatencyReport.writeToCSV(paths.filter(Files::isRegularFile) 56 | .filter(path -> path.toString().endsWith("output.log")) 57 | .flatMap(file -> { 58 | Map histograms = new HashMap<>(); 59 | final Matcher m = pattern.matcher(file.toString()); 60 | if (m.matches()) { 61 | var source = m.group(1); 62 | try { 63 | Files.readAllLines(file).forEach(line -> { 64 | Matcher matcher = LOG_PATTERN.matcher(line); 65 | if (matcher.matches()) { 66 | var destination = matcher.group(1); 67 | final SingleWriterRecorder histogram = histograms.computeIfAbsent(source + "<->" + destination, k -> new SingleWriterRecorder(Long.MAX_VALUE, 2)); 68 | double latency = Double.parseDouble(matcher.group(2)); 69 | String unit = matcher.group(4); 70 | long latencyInNanoseconds; 71 | if (unit.equals("ms")) { 72 | latencyInNanoseconds = (long) (latency * 1000 * 1000); 73 | } else { 74 | latencyInNanoseconds = (long) latency * 1000; 75 | } 76 | histogram.recordValue(latencyInNanoseconds); 77 | } 78 | }); 79 | return histograms.entrySet().stream().map(entry->{ 80 | var routeArr = entry.getKey().split("<->"); 81 | final LinkedHashMap row = LatencyTools.createLatencyReport(entry.getValue().getIntervalHistogram()); 82 | row.put("source", routeArr[0]); 83 | row.put("destination", routeArr[1]); 84 | return row; 85 | }); 86 | } catch (IOException e) { 87 | return null; 88 | } 89 | } else { 90 | return null; 91 | } 92 | }).filter(Objects::nonNull) 93 | .collect(toList()), 94 | "output.csv"); 95 | 96 | } catch (IOException e) { 97 | System.err.println("Error processing histogram log files: " + e.getMessage()); 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/com/aws/trading/StringMath.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * SPDX-License-Identifier: MIT-0 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 6 | * software and associated documentation files (the "Software"), to deal in the Software 7 | * without restriction, including without limitation the rights to use, copy, modify, 8 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 12 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 13 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 14 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 16 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | */ 18 | package com.aws.trading; 19 | 20 | import java.util.Arrays; 21 | 22 | public class StringMath { 23 | 24 | /** 25 | * The idea here is to multiply up the double by the scale, round it, then split it where the 26 | * decimal place should be. Where there are multiple leading zeros we fill these in and return 27 | * the string. Why not use BigDecimal you cry... because its shit. 28 | * Why not use the much simpler test version below? well it's twice as slow. 29 | * @param qty the double to be converted to string. 30 | * @param QTY_MULTIPLIER essentially 10 ^ SCALE 31 | * @param SCALE the number of supported decimal places in the qty or price. 32 | * @return A decimal string which is supported by the exchange. 33 | * */ 34 | public static String QtyToString(double qty, int SCALE, long QTY_MULTIPLIER){ 35 | final long qty_l = Math.round(qty * QTY_MULTIPLIER); 36 | final char[] qty_new = Long.toString(qty_l).toCharArray(); 37 | int len = qty_new.length; 38 | final char[] out; 39 | if(len > SCALE){ 40 | out = new char[len + 1]; 41 | final int loc = len - SCALE; 42 | System.arraycopy(qty_new, 0,out,0,loc); 43 | out[loc] = '.'; 44 | System.arraycopy(qty_new, loc,out,loc+1,len-loc); 45 | } else { 46 | out = new char[SCALE + 2]; 47 | out[0] = '0';out[1]='.'; 48 | final int loc = SCALE - len; 49 | if(loc > 0){ 50 | Arrays.fill(out,2,loc+2,'0'); 51 | } 52 | System.arraycopy(qty_new, 0,out,loc+2,len); 53 | } 54 | return new String(out); 55 | } 56 | 57 | public static String QtyToStringTest(double qty, int SCALE, double QTY_MULTIPLIER){ 58 | return Double.toString(Math.round(qty * QTY_MULTIPLIER)/QTY_MULTIPLIER); 59 | } 60 | 61 | public static long doubleToLong(final String input){ 62 | return Long.parseLong(input.replace(".","")); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/resources/config.properties: -------------------------------------------------------------------------------- 1 | COINPAIRS=BTC_USDT,BTC_CHF,BTC_EUR,BTC_USDC 2 | HOST=localhost 3 | HTTP_PORT=8888 4 | WEBSOCKET_PORT=8888 5 | USE_IOURING=false 6 | API_TOKEN=3001 7 | TEST_SIZE=50000 8 | EXCHANGE_CLIENT_COUNT=1 9 | WARMUP_COUNT=2 10 | USE_SSL=false 11 | KEY_STORE_PATH=/home/ec2-user/keystore.p12 12 | KEY_STORE_PASSWORD=123456 13 | # Each cipher should be specified individually, not as a colon-separated list 14 | CIPHERS=ECDHE-RSA-AES128-GCM-SHA256,ECDHE-RSA-AES256-GCM-SHA384,TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384 15 | PING_INTERVAL=60000 16 | -------------------------------------------------------------------------------- /src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | [%-5level] %d{UNIX_MILLIS} [%t] %c{1} - %msg%n 7 | > 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/test/java/ConfigTest.java: -------------------------------------------------------------------------------- 1 | import com.aws.trading.ExchangeProtocolImpl; 2 | 3 | import java.util.UUID; 4 | 5 | public class ConfigTest { 6 | public static void main(String[] args) { 7 | String instrument = "BTC/USDT"; 8 | var stringBuilder = new StringBuilder(); 9 | for(int i = 0; i< 300; i++){ 10 | stringBuilder.append(instrument); 11 | } 12 | System.out.println(new ExchangeProtocolImpl().createBuyOrder(stringBuilder.toString(), UUID.randomUUID().toString()).content().readableBytes()); 13 | System.out.println(stringBuilder.toString()); 14 | } 15 | } 16 | --------------------------------------------------------------------------------