├── .gitignore ├── LICENSE ├── README.md ├── infrastructure ├── aks │ ├── docker │ │ └── Dockerfile │ ├── lb.yaml │ └── setup_aks.sh ├── azfirewall.tf ├── data.tf ├── entropy.tf ├── linux_vm.tf ├── load_balancer_outbound.tf ├── load_balancer_server.tf ├── outputs.tf ├── route_table.tf ├── scripts │ ├── client.txt │ ├── manual_storage_account_cleanup.sh │ ├── manual_vmss_scaling.sh │ ├── my_ip.sh │ └── server.txt ├── service_bus.tf ├── storage_account_table.tf ├── terraform.tf ├── terraform.tfvars ├── variables.tf ├── vm_shutdown_schedule.tf ├── vmss-clients.tf ├── vmss-servers.tf ├── vnet-peering.tf └── vnet.tf ├── modules ├── generic_vm │ ├── main.tf │ ├── outputs.tf │ └── variables.tf └── vnet-peering │ ├── main.tf │ ├── outputs.tf │ └── variables.tf └── supplementals ├── HTTP.drawio ├── img ├── AFWEnableAccelnet.png ├── Topology0.png ├── Topology1.png ├── firewall policy.png └── spoke-to-hub.png ├── log.txt ├── stats ├── IDPS-network-delay-stats.xlsx ├── IDPS-network-stats.xlsx ├── http-latency-http-alert-vs-deny.png ├── latency-IDPS-Alert-D4-v4-iperf3-64-30-instances-5-min.png ├── latency-IDPS-Alert-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png ├── latency-IDPS-Alert-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png ├── latency-IDPS-Alert-and-Deny-D4-v4-iperf3-64-30-instances-5-min.png ├── latency-IDPS-Alert-and-Deny-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png ├── latency-IDPS-Alert-and-Deny-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png ├── latency-IDPS-Off-D4-v4-iperf3-64-30-instances-10-min.png ├── latency-IDPS-Off-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png ├── latency-IDPS-Off-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png ├── metrics-1740sec-30instances.png ├── metrics-280sec-30instances.png ├── metrics-30sec-30instances.png ├── metrics-540sec-120instances-idps-off.png ├── metrics-IDPS-Alert-D4-v4-iperf3-64-30-instances-5-min.png ├── metrics-IDPS-Alert-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png ├── metrics-IDPS-Alert-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png ├── metrics-IDPS-Alert-and-Deny-D4-v4-iperf3-64-30-instances-5-min.png ├── metrics-IDPS-Alert-and-Deny-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png ├── metrics-IDPS-Alert-and-Deny-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png ├── metrics-IDPS-Off-D4-v4-iperf3-64-30-instances-10-min.png ├── metrics-IDPS-Off-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png ├── metrics-IDPS-Off-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png ├── metrics-IDPS-vs-noIDPS-Alert-D4-v4-iperf3-64-30-instances-5-min.png ├── stats-1740sec-30instances.png ├── stats-1740sec-30instances.xlsx ├── stats-280sec-30instances.png ├── stats-280sec-30instances.xlsx ├── stats-30sec-30instances.png ├── stats-30sec-30instances.xlsx ├── stats-IDPS-Alert-D4-v4-iperf3-64-30-instances-5-min.csv ├── stats-IDPS-Alert-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.csv ├── stats-IDPS-Alert-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.csv ├── stats-IDPS-Alert-and-Deny-D4-v4-iperf3-64-30-instances-5-min.csv ├── stats-IDPS-Alert-and-Deny-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.csv ├── stats-IDPS-Alert-and-Deny-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.csv ├── stats-IDPS-Off-D4-v4-iperf3-64-30-instances-10-min.csv ├── stats-IDPS-Off-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.csv ├── stats-IDPS-Off-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.csv ├── stats-IDPS-on-off-table.png ├── stats-IDPS-on-off.png ├── stats-rtt-IDPS-on-off.png └── ~$IDPS-network-stats.xlsx └── topology.drawio /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | .vscode/* 3 | .vscode 4 | 5 | # Local .terraform directories 6 | **/.terraform/* 7 | 8 | # .tfstate files 9 | *.tfstate 10 | *.tfstate.* 11 | 12 | **/.terraform.lock.hcl 13 | .terraform.lock.hcl 14 | 15 | 16 | **/*.log 17 | *.log 18 | infrastructure/*.log 19 | 20 | # Crash log files 21 | crash.log 22 | 23 | # Ignore any .tfvars files that are generated automatically for each Terraform run. Most 24 | # .tfvars files are managed as part of configuration and so should be included in 25 | # version control. 26 | # 27 | # example.tfvars 28 | 29 | # Ignore override files as they are usually used to override resources locally and so 30 | # are not checked in 31 | override.tf 32 | override.tf.json 33 | *_override.tf 34 | *_override.tf.json 35 | 36 | # Include override files you do wish to add to version control using negated pattern 37 | # 38 | # !example_override.tf 39 | 40 | # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan 41 | # example: *tfplan* 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Andrew 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Azure Firewall premium throughput testing 2 | 3 | ## Overview 4 | 5 | The exercise aims to evaluate how the Azure Firewall Premium SKU performs under stress. A number of variables were measured, including throughput and delay for both IP and HTTP/HTTPS traffic. The performance of the Firewall was evaluated for different modes of IDPS - Alert, Alert and Deny, and Off. Some of the tests were performed without enabling AFWEnableAccelnet. 6 | 7 | In the lab, flows between spokes cross the firewall of the hub. VMSS served as a source and destination for the IP and HTTP(S) flows. VMSS provides on-demand scaling capabilities as well as the ability to control the maximum network bandwidth and maximum number of vCPUs per instance. 8 | 9 | In addition, the scenario of sources and destinations deployed on AKS was tested. We also tested scenarios where Standard Load Balancers were deployed in front of the "servers." Both did not outperform VMSS. 10 | 11 | The topology ( with slight variabions ) for the throughput testing is shown below. This is a classic hub-and-spoke topology with a deployment of Azure Firewall Premium in the hub. 12 | Deployment code ( terraform ) is in [this](infrastructure/) folder. 13 | 14 | The Firewall evaluated from the perspective of : 15 | 16 | 1. One flow performance ( AFWEnableAccelnet on/off ) 17 | 18 | 1. Combined IP performance ( IDPS Off) ( AFWEnableAccelnet on ) 19 | 20 | 1. IDPS related tests ( AFWEnableAccelnet on ): 21 | 22 | * HTTP(s) throughput ( IDPS Off, Alert, Alert and Deny ) 23 | 24 | * IP throughput ( IDPS Off, Alert , Alert and Deny ) 25 | 26 | * Relative RTT for IP ( IDPS Off, Alert, Alert and Deny) 27 | 28 | * Relative RTT for HTTP(s) ( IDPS Off, Alert, Alert and Deny) 29 | 30 | *) RTT is provided only as a relative measurement.See the note on RTT reporting in iperf3 and vegeta. 31 | 32 | 33 | ![Topology](supplementals/img/Topology0.png) 34 | 35 | 36 | There were four different VM sizes used, namely D4_v4, D5_v2, DS4_v2 and D48_v3. 37 | 38 | In comparison to the others, D4_v4 displays the best ratio of vCPU to network performance. Read more on performance data of VMs above [here](https://docs.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series) and [here](https://docs.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series) 39 | 40 | Summary of the VM performance: 41 | 42 | | VM size | vCPU | Expected Network bandwidth (Mbps) | Doc 43 | |-----------|---------|--------|-------| 44 | |D4_v4|4|10,000|https://docs.microsoft.com/en-us/azure/virtual-machines/dv4-dsv4-series 45 | |D5_v2|16|12,000|https://docs.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series 46 | |DS4_v2|8|6,000|https://docs.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series 47 | |D48_v3|48|24,000|https://docs.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series 48 | 49 | You should consider Azure quotas if you decide to repeat the tests - Azure subscription limits and quotas - https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/azure-subscription-service-limits 50 | 51 | Be sure to minimize Firewall transit traffic before changing the IDPS mode. You might see the following error otherwise. 52 | 53 | ![error](supplementals/img/firewall%20policy.png) 54 | 55 | ## Baseline tests. 56 | 57 | Learn more about Azure Firewall Performance Boost here - https://docs.microsoft.com/en-us/azure/firewall/firewall-performance. 58 | 59 | Below you will find the results of Iperf3 tests over the Azure Firewall Premium, with the AFWEnableAccelnet is set to False. 60 | 61 | ### iperf3 spoke1 to spoke2 ( no tunneling ) 62 | 63 | One Flow (DS4_v2 with AN) 64 | ``` 65 | azureadmin@spoke1-vm:~$ iperf3 -c 10.1.0.4 66 | Connecting to host 10.1.0.4, port 5201 67 | [ 4] local 10.2.0.4 port 45070 connected to 10.1.0.4 port 5201 68 | 69 | - - - - - - - - - - - - - - - - - - - - - - - - - 70 | [ ID] Interval Transfer Bandwidth Retr 71 | [ 4] 0.00-10.00 sec 2.30 GBytes 1.98 Gbits/sec 385 sender 72 | [ 4] 0.00-10.00 sec 2.30 GBytes 1.97 Gbits/sec receiver 73 | ``` 74 | 75 | 64 parallel flows (-P64): (DS4_v2 with AN) 76 | ``` 77 | 78 | [SUM] 0.00-10.00 sec 6.50 GBytes 5.58 Gbits/sec 6 sender 79 | [SUM] 0.00-10.00 sec 6.44 GBytes 5.53 Gbits/sec receiver 80 | 81 | ``` 82 | ### iperf3 spoke1 to hub (bypassing firewall) 83 | 84 | One Flow (DS4_v2 with AN) - There is no doubt that total bandwidth (for both tests) is close to the maximum network performance of the VM. 85 | ``` 86 | azureadmin@spoke1-vm:~$ iperf3 -c 10.0.1.4 87 | Connecting to host 10.0.1.4, port 5201 88 | [ 4] local 10.2.0.4 port 53928 connected to 10.0.1.4 port 5201 89 | [ ID] Interval Transfer Bandwidth Retr Cwnd 90 | - - - - - - - - - - - - - - - - - - - - - - - - - 91 | [ ID] Interval Transfer Bandwidth Retr 92 | [ 4] 0.00-10.00 sec 6.66 GBytes 5.72 Gbits/sec 0 sender 93 | [ 4] 0.00-10.00 sec 6.65 GBytes 5.72 Gbits/sec receiver 94 | 95 | iperf Done. 96 | ``` 97 | 64 parallel flows (-P64): (DS4_v2 with AN) 98 | 99 | ``` 100 | [SUM] 0.00-10.00 sec 6.73 GBytes 5.78 Gbits/sec 0 sender 101 | [SUM] 0.00-10.00 sec 6.66 GBytes 5.72 Gbits/sec receiver 102 | ``` 103 | 104 | ## One Flow performance 105 | ### Combining flows with vxlan encapsulation 106 | 107 | The best way to combine multiple flows is to wrap them with Vxlan encapsulation: (IPIP and GRE are not permitted in Azure) 108 | 109 | ``` 110 | # spoke1 111 | ip link add vxlan1 type vxlan id 1 remote 10.1.0.4 dstport 4789 dev eth0 112 | ip link set vxlan1 up 113 | ip addr add 10.77.0.1/30 dev vxlan1 114 | 115 | # spoke2 116 | 117 | ip link add vxlan1 type vxlan id 1 remote 10.2.0.4 dstport 4789 dev eth0 118 | ip link set vxlan1 up 119 | ip addr add 10.77.0.2/30 dev vxlan1 120 | ``` 121 | 122 | ``` 123 | # spoke to spoke over VXLAN : 64 flows DS4_v2 with AN ; The performance of the session fluctuates between 2.5 Gbps, as expected. 124 | 125 | root@spoke1-vm:~# iperf3 -P 64 -c 10.77.0.2 126 | [SUM] 0.00-10.00 sec 2.89 GBytes 2.48 Gbits/sec 214 sender 127 | [SUM] 0.00-10.00 sec 2.77 GBytes 2.38 Gbits/sec receiver 128 | 129 | root@spoke1-vm:~# iperf3 -P 128 -c 10.77.0.2 130 | [SUM] 0.00-10.00 sec 2.92 GBytes 2.51 Gbits/sec 1144 sender 131 | [SUM] 0.00-10.00 sec 2.78 GBytes 2.39 Gbits/sec receiver 132 | ``` 133 | 134 | ``` 135 | #spoke to spoke over VXLAN : 64 flows : DS5_v2 with AN 136 | [SUM] 0.00-10.00 sec 2.17 GBytes 1.87 Gbits/sec 500 sender 137 | [SUM] 0.00-10.00 sec 2.03 GBytes 1.74 Gbits/sec receiver 138 | 139 | [SUM] 0.00-10.00 sec 2.26 GBytes 1.94 Gbits/sec 1814 sender 140 | [SUM] 0.00-10.00 sec 2.13 GBytes 1.83 Gbits/sec receiver 141 | ``` 142 | 143 | ### Register AFWEnableAccelnet feature 144 | 145 | To enable the Azure Firewall Premium performance boost, run the following Azure PowerShell commands. This feature is applied at the subscription level for all Firewalls (VNet Firewalls and SecureHub Firewalls). More [here](https://docs.microsoft.com/en-us/azure/firewall/firewall-performance) and [here](https://docs.microsoft.com/en-us/azure/firewall/firewall-preview) 146 | 147 | 148 | Commands are run in Azure PowerShell to enable the features. For the feature to immediately take effect, an operation needs to be run on the firewall. This can be a rule change (least intrusive), a setting change, or a stop/start operation. Otherwise, the firewall/s is updated with the feature within several days. 149 | 150 | ``` 151 | Select-AzSubscription -Subscription 152 | 153 | Register-AzProviderFeature -Featurename AFWEnableAccelnet -ProviderNamespace Microsoft.Network 154 | 155 | ``` 156 | ![AFWEnableAccelnet](/supplementals/img/AFWEnableAccelnet.png) 157 | 158 | Re-testing with VXLAN encapsulation. Spoke to Spoke : 159 | 160 | ``` 161 | # spoke to spoke over VXLAN : 64 flows DS5_v2 with AN 162 | 163 | [SUM] 0.00-10.00 sec 6.07 GBytes 5.21 Gbits/sec 2868 sender 164 | [SUM] 0.00-10.00 sec 5.94 GBytes 5.10 Gbits/sec receiver 165 | 166 | [SUM] 0.00-10.00 sec 5.70 GBytes 4.90 Gbits/sec 496 sender 167 | [SUM] 0.00-10.00 sec 5.60 GBytes 4.81 Gbits/sec receiver 168 | 169 | [SUM] 0.00-10.00 sec 5.39 GBytes 4.63 Gbits/sec 38 sender 170 | [SUM] 0.00-10.00 sec 5.30 GBytes 4.55 Gbits/sec receiver 171 | 172 | 173 | # spoke to spoke over VXLAN : 64 flows DS4_v2 with AN 174 | 175 | [SUM] 0.00-10.00 sec 5.70 GBytes 4.90 Gbits/sec 496 sender 176 | [SUM] 0.00-10.00 sec 5.60 GBytes 4.81 Gbits/sec receiver 177 | 178 | [SUM] 0.00-10.00 sec 6.64 GBytes 5.71 Gbits/sec 4840 sender 179 | [SUM] 0.00-10.00 sec 6.61 GBytes 5.68 Gbits/sec receiver 180 | 181 | [SUM] 0.00-10.00 sec 6.64 GBytes 5.71 Gbits/sec 4840 sender 182 | [SUM] 0.00-10.00 sec 6.61 GBytes 5.68 Gbits/sec receiver 183 | 184 | 185 | ``` 186 | 187 | Throughput between hub and spoke directly (bypassing Firewall) over VXLAN. 188 | 189 | 190 | ![spoke to hub](supplementals/img/spoke-to-hub.png) 191 | 192 | Configuration 193 | 194 | ``` 195 | # spoke1 196 | ip link add vxlan2 type vxlan id 2 remote 10.0.1.4 dstport 4789 dev eth0 197 | ip link set vxlan2 up 198 | ip addr add 10.78.0.1/30 dev vxlan2 199 | 200 | # hub 201 | 202 | ip link add vxlan2 type vxlan id 2 remote 10.2.0.4 dstport 4789 dev eth0 203 | ip link set vxlan2 up 204 | ip addr add 10.78.0.2/30 dev vxlan2 205 | ``` 206 | 207 | Results 208 | 209 | ``` 210 | on DS4_v2 with AN spoke to hub , bypassing firewall . vxlan 1 to vxlan 2 , 64 flows 211 | 212 | [SUM] 0.00-10.00 sec 6.52 GBytes 5.60 Gbits/sec 0 sender 213 | [SUM] 0.00-10.00 sec 6.44 GBytes 5.53 Gbits/sec receiver 214 | 215 | ``` 216 | 217 | What does the above result mean? 218 | 219 | When AFWEnableAccelnet is enabled, the performance of the single flow reaches the maximum allowed by the virtual machine. 220 | 221 | ### Test based on Standard_D48_v3 222 | 223 | Summary : The max performance of one flow is close to 10Gbps. Even on VMs that can handle 24Gbps, a single transit flow through Azure Firewall is limited to 10Gbps. This is 4x better than 2.5Gbps Firewall performance with AFWEnableAccelnet is disabled. 224 | 225 | | Test Case | VM size | Src | Dst | Encap | Proto | Flows | Performance | Note | 226 | |-----------|---------|--------|-------------|---------------|----------| :-: | :-: | :-: | 227 | | 1. One flow iperf3 spoke to spoke | Standard_D48_v3 [24 Mbps](https://docs.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series) | spoke1 (10.2.0.4) | spoke2 (10.1.0.4) | native | iperf3 | 1 | 9.33 Gbits/sec | [linky](#test-case-1---iperf3-one-flow) 228 | | 2. 64 flows iperf3 spoke to spoke | Standard_D48_v3 [24 Mbps](https://docs.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series) | spoke1 (10.2.0.4) | spoke2 (10.1.0.4) | native | iperf3 | 64 | 13.9 - 9.55 Gbit/sec| [linky](#test-case-2---64-flows-iperf3-spoke-to-spoke) 229 | | 3. One flow iperf3 spoke to spoke encapsulated to VXLAN | Standard_D48_v3 [24 Mbps](https://docs.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series) | spoke1 (10.77.0.1) | spoke2 (10.77.0.2) | VXLAN | iperf3 | 1 | 1.33 Gbits/sec | [linky](#test-case-3---one-flow-spoke-to-spoke-iperf3-encapsulated-to-vxlan) 230 | | 4. 64 flows iperf3 spoke to spoke encapsulated to VXLAN | Standard_D48_v3 [24 Mbps](https://docs.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series) | spoke1 (10.77.0.1) | spoke2 (10.77.0.2) | VXLAN | iperf3 | 64 | 11.6 Gbits/sec | [linky](#test-case-4---64-flows-spoke-to-spoke-iperf3-encapsulated-to-vxlan) 231 | | 5. One flow ftp spoke to spoke | Standard_D48_v3 [24 Mbps](https://docs.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series) | spoke1 (10.2.0.4) | spoke2 (10.1.0.4) | native | ftp | 1 | 310.38 MB/s = 2480 Mbps | [linky](#test-case-5---one-flow-spoke-to-spoke-ftp) 232 | | 6. 64 flows lftp spoke to spoke encapsulated to VXLAN | Standard_D48_v3 [24 Mbps](https://docs.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series) | spoke1 (10.77.0.1) | spoke2 (10.77.0.2) | VXLAN | lftp | 64 | 300 MiB/sec = 2516 Mbps| [linky](#test-case-6---64-flows-spoke-to-spoke-lftp-encapsulated-to-vxlan) 233 | | 7. One flow iperf3 spoke to spoke | Standard_DS5_v2 [12 Mbps](https://docs.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series) | spoke1 (10.2.0.4) | spoke2 (10.1.0.4) | native | iperf3 | 1 | 9.42 Gbps/s | [linky](#test-case-7-one-flow-iperf3-spoke-to-spoke) 234 | | 8. 64 flows iperf3 spoke to spoke | Standard_DS5_v2 [12 Mbps](https://docs.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series) | spoke1 (10.2.0.4) | spoke2 (10.1.0.4) | native | iperf3 | 64 | 10.9 Gbps/s| [linky](#test-case-8-64-flows-iperf3-spoke-to-spoke) 235 | | 9. One flow iperf3 spoke to spoke encapsulated to VXLAN |Standard_DS5_v2 [12 Mbps](https://docs.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series)| spoke1 (10.77.0.1) | spoke2 (10.77.0.2) | VXLAN | iperf3 | 1 | 1.37 Gbps/s | [linky](#test-case-9-one-flow-iperf3-spoke-to-spoke-encapsulated-to-vxlan) 236 | | 10. 64 flows iperf3 spoke to spoke encapsulated to VXLAN | Standard_DS5_v2 [12 Mbps](https://docs.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series) | spoke1 (10.77.0.1) | spoke2 (10.77.0.2) | VXLAN | iperf3 | 64 | 10.7 Gbps/s | [linky](#test-case-10-64-flows-iperf3-spoke-to-spoke-encapsulated-to-vxlan) 237 | 238 | How would you interpret the above result? 239 | 240 | When AFWEnableAccelnet is enabled, the performance of the single flow reaches the 10Gbps. 241 | 242 | Further details about the tests summarized in the above table can be found below 243 | 244 | ### Test case 1 - One flow iperf3 spoke to spoke 245 | 246 | spoke1 247 | ``` 248 | iperf3 -c 10.1.0.4 249 | 250 | [ ID] Interval Transfer Bandwidth Retr 251 | [ 4] 0.00-10.00 sec 10.9 GBytes 9.33 Gbits/sec 1208 sender 252 | [ 4] 0.00-10.00 sec 10.9 GBytes 9.33 Gbits/sec receiver 253 | 254 | [ 4] 0.00-10.00 sec 11.0 GBytes 9.42 Gbits/sec 621 sender 255 | [ 4] 0.00-10.00 sec 11.0 GBytes 9.42 Gbits/sec receiver 256 | ``` 257 | spoke2 258 | ``` 259 | iperf3 -s 260 | ``` 261 | 262 | ### Test case 2 - 64 flows iperf3 spoke to spoke 263 | 264 | spoke1 265 | ``` 266 | iperf3 -c 10.1.0.4 -P 64 267 | 268 | [SUM] 0.00-10.00 sec 16.2 GBytes 13.9 Gbits/sec 6198 sender 269 | [SUM] 0.00-10.00 sec 16.1 GBytes 13.8 Gbits/sec receiver 270 | 271 | [SUM] 0.00-10.00 sec 11.1 GBytes 9.55 Gbits/sec 10512 sender 272 | [SUM] 0.00-10.00 sec 11.0 GBytes 9.46 Gbits/sec receiver 273 | 274 | [SUM] 0.00-10.00 sec 11.1 GBytes 9.53 Gbits/sec 17613 sender 275 | [SUM] 0.00-10.00 sec 11.0 GBytes 9.47 Gbits/sec receiver 276 | 277 | ``` 278 | spoke2 279 | ``` 280 | iperf3 -s 281 | ``` 282 | 283 | ### Test case 3 - One flow spoke to spoke iperf3 encapsulated to VXLAN 284 | spoke1 285 | ``` 286 | ip link add vxlan1 type vxlan id 1 remote 10.1.0.4 dstport 4789 dev eth0 287 | ip link set vxlan1 up 288 | ip addr add 10.77.0.1/30 dev vxlan1 289 | 290 | iperf3 -c 10.77.0.2 291 | 292 | [ ID] Interval Transfer Bandwidth Retr 293 | [ 4] 0.00-10.00 sec 1.25 GBytes 1.08 Gbits/sec 62 sender 294 | [ 4] 0.00-10.00 sec 1.25 GBytes 1.08 Gbits/sec receiver 295 | 296 | [ ID] Interval Transfer Bandwidth Retr 297 | [ 4] 0.00-10.00 sec 1.55 GBytes 1.33 Gbits/sec 90 sender 298 | [ 4] 0.00-10.00 sec 1.55 GBytes 1.33 Gbits/sec receiver 299 | ``` 300 | 301 | 302 | spoke2 303 | ``` 304 | ip link add vxlan1 type vxlan id 1 remote 10.2.0.4 dstport 4789 dev eth0 305 | ip link set vxlan1 up 306 | ip addr add 10.77.0.2/30 dev vxlan1 307 | 308 | iperf3 -s -i vxlan1 309 | ``` 310 | 311 | ### Test case 4 - 64 flows spoke to spoke iperf3 encapsulated to VXLAN 312 | spoke1 313 | ``` 314 | ip link add vxlan1 type vxlan id 1 remote 10.1.0.4 dstport 4789 dev eth0 315 | ip link set vxlan1 up 316 | ip addr add 10.77.0.1/30 dev vxlan1 317 | 318 | iperf3 -c 10.77.0.2 -P64 319 | 320 | [SUM] 0.00-10.04 sec 11.0 GBytes 9.38 Gbits/sec 5267 sender 321 | [SUM] 0.00-10.04 sec 10.9 GBytes 9.35 Gbits/sec receiver 322 | 323 | [SUM] 0.00-10.00 sec 8.25 GBytes 7.08 Gbits/sec 1747 sender 324 | [SUM] 0.00-10.00 sec 8.25 GBytes 7.08 Gbits/sec receiver 325 | 326 | [SUM] 0.00-10.03 sec 13.5 GBytes 11.6 Gbits/sec 4488 sender 327 | [SUM] 0.00-10.03 sec 13.3 GBytes 11.4 Gbits/sec receiver 328 | ``` 329 | 330 | spoke2 331 | ``` 332 | ip link add vxlan1 type vxlan id 1 remote 10.2.0.4 dstport 4789 dev eth0 333 | ip link set vxlan1 up 334 | ip addr add 10.77.0.2/30 dev vxlan1 335 | 336 | iperf3 -s -i vxlan1 337 | ``` 338 | 339 | ### Test case 5 - One flow spoke to spoke ftp 340 | spoke1 341 | ``` 342 | apt-get install ncftp 343 | 344 | ncftp / > get 10gb-file 345 | 10gb-file: 9.31 GB 103.35 MB/s 346 | ncftp / > get 10gb-file 347 | 10gb-file: 9.31 GB 310.38 MB/s 348 | ``` 349 | spoke2 350 | ``` 351 | apt-get install proftpd 352 | # enable anonymous access ( /etc/proftpd/proftpd.conf ) 353 | cd /srv/ftp 354 | dd if=/dev/urandom of=10gb-file bs=1000000000 count=10 iflag=fullblock # takes about 2-3 min 355 | # root@spoke2-vm:/srv/ftp# ls -lah 356 | # total 9.4G 357 | # drwxr-xr-x 2 ftp nogroup 4.0K Dec 22 14:04 . 358 | # drwxr-xr-x 3 root root 4.0K Dec 22 13:50 .. 359 | # -rw-r--r-- 1 root root 9.4G Dec 22 14:08 10gb-file 360 | # -rw-r--r-- 1 root root 170 Mar 12 2018 welcome.msg 361 | ``` 362 | 363 | ### Test case 6 - 64 flows spoke to spoke lftp encapsulated to VXLAN 364 | 365 | Iperf3 is, in fact, the best performance testing tool. It is extremely difficult to fill up a pipe completely using ftp. 366 | 367 | spoke1 368 | ``` 369 | lftp -e 'pget -n 64 ftp://10.77.0.2/10gb-file;quit' 370 | 371 | 10000000000 bytes transferred in 31 seconds (305.19 MiB/s) 372 | 373 | ``` 374 | 375 | ### Test case 7. One flow iperf3 spoke to spoke 376 | ``` 377 | - - - - - - - - - - - - - - - - - - - - - - - - - 378 | [ ID] Interval Transfer Bandwidth Retr 379 | [ 4] 0.00-10.00 sec 10.8 GBytes 9.30 Gbits/sec 2487 sender 380 | [ 4] 0.00-10.00 sec 10.8 GBytes 9.29 Gbits/sec receiver 381 | 382 | [ ID] Interval Transfer Bandwidth Retr 383 | [ 4] 0.00-10.00 sec 11.0 GBytes 9.42 Gbits/sec 1410 sender 384 | [ 4] 0.00-10.00 sec 11.0 GBytes 9.41 Gbits/sec receiver 385 | ``` 386 | ### Test case 8. 64 flows iperf3 spoke to spoke 387 | 388 | ``` 389 | [SUM] 0.00-10.00 sec 12.7 GBytes 10.9 Gbits/sec 13621 sender 390 | [SUM] 0.00-10.00 sec 12.6 GBytes 10.9 Gbits/sec receiver 391 | 392 | [SUM] 0.00-10.00 sec 12.7 GBytes 10.9 Gbits/sec 6427 sender 393 | [SUM] 0.00-10.00 sec 12.7 GBytes 10.9 Gbits/sec receiver 394 | ``` 395 | 396 | ### Test case 9. One flow iperf3 spoke to spoke encapsulated to VXLAN 397 | ``` 398 | [ ID] Interval Transfer Bandwidth Retr 399 | [ 4] 0.00-10.00 sec 1.59 GBytes 1.37 Gbits/sec 84 sender 400 | [ 4] 0.00-10.00 sec 1.59 GBytes 1.37 Gbits/sec receiver 401 | 402 | [ ID] Interval Transfer Bandwidth Retr 403 | [ 4] 0.00-10.00 sec 1.59 GBytes 1.37 Gbits/sec 84 sender 404 | [ 4] 0.00-10.00 sec 1.59 GBytes 1.37 Gbits/sec receiver 405 | ``` 406 | ### Test case 10. 64 flows iperf3 spoke to spoke encapsulated to VXLAN 407 | ``` 408 | [SUM] 0.00-10.00 sec 12.5 GBytes 10.7 Gbits/sec 17549 sender 409 | [SUM] 0.00-10.00 sec 12.4 GBytes 10.6 Gbits/sec receiver 410 | ``` 411 | 412 | ## Combined throughput on max-pre-scaled Firewall Premium. 413 | 414 | The purpose of the test is to max out the Firewall's combined performance. The following three methods were used in order to verify the performance of the Firewall Premium prescaled to 20 instances: 415 | 416 | 1. iperf3 30 sec, short-duration flows in VMSS , combined over 30 instances 417 | 2. iperf3 280 sec , longer-duration flows, with 64x multiplier , combined over 30 instances. Compared with AZFW Throughput Metrics 418 | 3. iperf3 29 min (1740 sec) , jumbo flows, with 64x multiplier , combined over 30 instances. Compared with AZFW Throughput Metrics 419 | 420 | All three methods above provide an average 180Gbps performance with unsignificant devaiation. 421 | 422 | Topology below were used for test: 423 | 424 | ![VMSSTopology](supplementals/img/Topology1.png) 425 | 426 | Notes about implementation: 427 | 428 | 1. Client/Server architecture were used 429 | 2. Clients/Servers were implemented as VMSS 430 | 3. Clients and Servers are NTP-synchronized 431 | 4. Azure Service Bus queue were used for feeding client pool with server IP addresses 432 | 5. Azure Table Store used for statistics collection 433 | 6. VMSS were built based on [Standard_D4_v4](https://docs.microsoft.com/en-us/azure/virtual-machines/dv4-dsv4-series) ( 4 cores, 10Gbps network performance ) 434 | 7. 30 instances should be enough to cover potential max of the AZFW ( 20 * 10Gbps < 30 * 10Gbps) 435 | 436 | ### iperf3 30 sec, short-duration flows in VMSS , combined over 30 instances 437 | 438 | [Statistics collected from 30 clients](supplementals/stats/stats-30sec-30instances.xlsx) 439 | 440 | Visualization ( aggregate ): 441 | 442 | ![Visualization](supplementals/stats/stats-30sec-30instances.png) 443 | 444 | 445 | ### iperf3 280 sec , longer-duration flows, with 64x multiplier , combined over 30 instances. Compared with AZFW Throughput Metrics 446 | 447 | [Statistics collected from 30 clients](supplementals/stats/stats-280sec-30instances.xlsx) 448 | 449 | Visualization ( aggregate ), Note the throughput is symmetrical 450 | 451 | ![Visualization](supplementals/stats/stats-280sec-30instances.png) 452 | 453 | Metrics ( AZFW Throughput ) 454 | 455 | ![Metrics](supplementals/stats/metrics-280sec-30instances.png) 456 | 457 | ### iperf3 29 min (1740 sec) , jumbo flows, with 64x multiplier , combined over 30 instances. Compared with AZFW Throughput Metrics 458 | 459 | [Statistics collected from 30 clients](supplementals/stats/stats-1740sec-30instances.xlsx) 460 | 461 | Visualization ( aggregate ), Note the throughput is symmetrical 462 | 463 | ![Visualization](supplementals/stats/stats-1740sec-30instances.png) 464 | 465 | ![Metrics](supplementals/stats/metrics-1740sec-30instances.png) 466 | 467 | ## RTT measurements in iperf3 and vegeta 468 | 469 | RTT data is only provided as a relative and comparative measure. In a hub-spoke environment with IDPS off, round trip time from VM to VM is 2.35 milliseconds. 470 | 471 | Iperf3 and vegeta use different reporting methods. Additionally, server processing delays need to be taken into consideration. Accordingly, you should only use them relative to the other measurements I mentioned above. 472 | For example, you can compare RTT for transit traffic with IDPS OFF versus IDPS Alert and Deny and see the percentage of change. Take the scale into consideration! 473 | 474 | ![comparsion](supplementals/stats/http-latency-http-alert-vs-deny.png) 475 | 476 | ### Iperf3 latency calculation/reporting 477 | 478 | As per [esnet/iperf](https://github.com/esnet/iperf/commit/432ef7ebb3abfedcb87b717b251eb72fc1a2d0c3) : 479 | 480 | >Retrieve RTT information on platforms supporting it. 481 | This value is available on the sender side, expressed in 482 | microseconds. It's available in the JSON output. 483 | 484 | >In the JSON output we also output the maximum observed RTT 485 | per-stream. Note that since the observation interval is many times 486 | the RTT, it's not clear how good this value would be at capturing the 487 | largest computed RTT value over the lifetime of each stream. 488 | 489 | >While here, also determine the maximum observed snd_cwnd value over 490 | the lifetime of each stream. 491 | 492 | >This all works pretty well on Linux, but on FreeBSD (which should 493 | theoretically be supported) we don't do a good job of supporting the 494 | tcp_info structure. We need to make this code a lot more portable, 495 | rather than just assuming the world of platforms is "Linux" 496 | vs. "everything else". Fixing this requires some rearchitecting of 497 | the way that we retrieve, compute, and print statistics. 498 | 499 | RTT returned in [usec](https://github.com/esnet/iperf/blob/332c31ee6512514c216077407a725b5b958b1582/src/tcp_info.c#L168) 500 | 501 | Why IPERF3 reported latency is differs from icmp/ping ? 502 | > 503 | >This is due to the standard package size in iperf3, in TCP this is 8KB, 1470 bytes for UDP. When using ping it is 56 bytes by default. Try specifying the --length flag in iperf3 and you will see similar RTT times. Note that you will never get exactly the same result because ping uses ICMP (which is a network layer protocol), whereas iperf3 is on the transport layer. 504 | 505 | [orig](https://github.com/esnet/iperf/issues/635#issuecomment-1022254224) 506 | > 507 | ``` 508 | azureadmin@spoke1-vm:~$ ping 10.1.0.4 509 | PING 10.1.0.4 (10.1.0.4) 56(84) bytes of data. 510 | 64 bytes from 10.1.0.4: icmp_seq=1 ttl=63 time=4.52 ms 511 | 64 bytes from 10.1.0.4: icmp_seq=2 ttl=63 time=2.35 ms 512 | 64 bytes from 10.1.0.4: icmp_seq=3 ttl=63 time=2.34 ms 513 | 64 bytes from 10.1.0.4: icmp_seq=4 ttl=63 time=2.26 ms 514 | ^C 515 | --- 10.1.0.4 ping statistics --- 516 | 4 packets transmitted, 4 received, 0% packet loss, time 3004ms 517 | rtt min/avg/max/mdev = 2.265/2.874/4.529/0.956 ms 518 | 519 | Reported by iperf3 : ( iperf3 --length 56 -P64 -t 180 -c 10.1.0.4 -J ) 520 | 521 | "max_snd_cwnd": 57288, 522 | "max_rtt": 8092, 523 | "min_rtt": 478, 524 | "mean_rtt": 1290 525 | 526 | ``` 527 | ### Vegeta latency calculation/reporting 528 | 529 | To convert vegeta-reported lateny to ms divide the result by 1,000,000 530 | 531 | ``` 532 | echo "GET http://10.1.0.5" | vegeta attack -duration=10s -insecure -rate=50 | vegeta report 533 | Latencies [min, mean, 50, 90, 95, 99, max] 53.076ms, 65.005ms, 62.132ms, 69.608ms, 79.67ms, 112.85ms, 192.376ms 534 | 535 | { 536 | "latencies": { 537 | "total": 38415392406, 538 | "mean": 76830784, 539 | "50th": 64352932, 540 | "90th": 109928619, 541 | "95th": 117367639, 542 | "99th": 206221839, 543 | "max": 374886829, 544 | "min": 50137960 545 | } 546 | 547 | 65ms ~~ 76830784/1000000 548 | ``` 549 | 550 | ## Throughput and RTT for HTTP(s) and IP traffic with different IDPS modes 551 | 552 | 553 | We used vegeta ( https://github.com/tsenart/vegeta ) and nginx ( https://github.com/nginx/nginx ) in client/server scenario for load generation. VMSS of different sizes ( 40 - 120 ) used for testing. 554 | 555 | Vegeta used with the rate set to 50 for http(s), nginx generate 1Mb random file. 556 | 557 | In summary, with IDPS off, the http(s) throughput is proportional to the size of VMSS, but with IDPS set to Alert and Deny, the throughput dropped to 9Gbps. With Alert, performance was decreased by 10% compared to performance of the firewall with IDPS off. Read more below. 558 | 559 | Additionally, we have tested scenarios where source/destinations are set up on AKS clusters. We also tested the scenario with a standard load balancer in front of the servers. Both of them have not provided any performance benefits, despite rebalancing sources and destinations. 560 | 561 | 562 | ### IDPS-Off-D4-v4-vegeta-http-rate-50-1M-reply-240-instances-5-min 563 | 564 | Variables: 565 | ``` 566 | # terraform.tfvars: 567 | idps = "Off" 568 | 569 | vmss_size = "Standard_D2_v4" 570 | 571 | # clients.txt: 572 | test_type = "vegeta" 573 | duration = 5 # in minutes 574 | protocol = "http" # vegeta specific 575 | 576 | 577 | ./manual_vmss_scaling.sh 240 # az cli scaling is much faster than with terraform 578 | ``` 579 | 580 | * Throughput : ~ 54Gbps 581 | * Latency mean : 209.7ms ( include server processing delay ) 582 | 583 | [stats](/supplementals/stats/stats-IDPS-Off-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.csv) 584 | ![Metrics](/supplementals/stats/metrics-IDPS-Off-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png) 585 | ![Latency](/supplementals/stats/latency-IDPS-Off-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png) 586 | ### IDPS-Off-D4-v4-vegeta-https-rate-50-1M-reply-240-instances-5-min 587 | 588 | Variables: 589 | ``` 590 | # terraform.tfvars: 591 | idps = "Off" 592 | 593 | vmss_size = "Standard_D2_v4" 594 | 595 | testtype = "vegeta" 596 | testduration = 5 597 | testprotocol = "https" 598 | 599 | 600 | ./manual_vmss_scaling.sh 240 # az cli scaling is much faster than with terraform 601 | ``` 602 | 603 | * Throughput : ~52Gbps 604 | * Latency mean : 227.16ms ( include server processing delay ) 605 | 606 | [stats](/supplementals/stats/stats-IDPS-Off-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.csv) 607 | ![Metrics](/supplementals/stats/metrics-IDPS-Off-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png) 608 | ![Latency](/supplementals/stats/latency-IDPS-Off-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png) 609 | ### IDPS-Off-D4-v4-iperf3-64-30-instances-5-min 610 | 611 | Variables: 612 | ``` 613 | 614 | # terraform.tfvars: 615 | idps = "Off" 616 | testtype = "iperf3" 617 | 618 | vmss_size = "Standard_D4_v4" 619 | 620 | testduration = 5 621 | testprotocol = "" 622 | testiperf3flows = "64" 623 | 624 | 625 | ./manual_vmss_scaling.sh 30 # az cli scaling is much faster than with terraform 626 | ``` 627 | 628 | * Throughput Gbps: 140Gbps 629 | * Latency mean : 36854.53571 630 | 631 | [stats](/supplementals/stats/stats-IDPS-Off-D4-v4-iperf3-64-30-instances-10-min.csv) 632 | ![Metrics](/supplementals/stats/metrics-IDPS-Off-D4-v4-iperf3-64-30-instances-10-min.png) 633 | ![Latency](/supplementals/stats/latency-IDPS-Off-D4-v4-iperf3-64-30-instances-10-min.png) 634 | ### IDPS-Alert-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min 635 | 636 | Variables: 637 | ``` 638 | # terraform.tfvars: 639 | idps = "Alert" 640 | testtype = "vegeta" 641 | testduration = 5 642 | testprotocol = "http" 643 | 644 | ./manual_vmss_scaling.sh 120 # az cli scaling is much faster than with terraform 645 | ``` 646 | 647 | * Throughput : ~45Gbps 648 | * Latency mean : 14359.77058 ( include server processing delay ) 649 | 650 | [stats](/supplementals/stats/stats-IDPS-Alert-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.csv) 651 | ![Metrics](/supplementals/stats/metrics-IDPS-Alert-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png) 652 | ![Latency](/supplementals/stats/latency-IDPS-Alert-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png) 653 | ### IDPS-Alert-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min 654 | 655 | Variables: 656 | ``` 657 | # terraform.tfvars: 658 | idps = "Alert" 659 | testtype = "vegeta" 660 | testduration = 5 661 | testprotocol = "https" 662 | 663 | ./manual_vmss_scaling.sh 120 # az cli scaling is much faster than with terraform 664 | ``` 665 | 666 | * Throughput : ~54 Gbps 667 | * Latency mean : 2281.117052 ( include server processing delay ) 668 | 669 | [stats](/supplementals/stats/stats-IDPS-Alert-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.csv) 670 | ![Metrics](/supplementals/stats/metrics-IDPS-Alert-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png) 671 | ![Latency](/supplementals/stats/latency-IDPS-Alert-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png) 672 | ### IDPS-Alert-D4-v4-iperf3-64-30-instances-5-min 673 | 674 | Variables: 675 | ``` 676 | # terraform.tfvars: 677 | idps = "Alert" 678 | testtype = "iperf3" 679 | 680 | vmss_size = "Standard_D4_v4" 681 | 682 | testduration = 5 683 | testprotocol = "" 684 | testiperf3flows = "64" 685 | 686 | 687 | ./manual_vmss_scaling.sh 30 # az cli scaling is much faster than with terraform 688 | ``` 689 | * Throughput : ~ 55Gbps 690 | * Latency mean : 76472.87538 ( include server processing delay ) 691 | 692 | [stats](/supplementals/stats/stats-IDPS-Alert-D4-v4-iperf3-64-30-instances-5-min.csv) 693 | ![Metrics](/supplementals/stats/metrics-IDPS-Alert-D4-v4-iperf3-64-30-instances-5-min.png) 694 | ![Latency](/supplementals/stats/latency-IDPS-Alert-D4-v4-iperf3-64-30-instances-5-min.png) 695 | ![Comparison](/supplementals/stats/metrics-IDPS-vs-noIDPS-Alert-D4-v4-iperf3-64-30-instances-5-min.png) 696 | ### IDPS-Alert-and-Deny-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min 697 | 698 | Variables: 699 | ``` 700 | # terraform.tfvars: 701 | idps = "Deny" 702 | testtype = "vegeta" 703 | testduration = 5 704 | testprotocol = "http" 705 | 706 | 707 | ./manual_vmss_scaling.sh 120 # az cli scaling is much faster than with terraform 708 | ``` 709 | 710 | * Throughput : ~ 9.7 Gbps 711 | * Latency mean : 29801.40659 ( include server processing delay ) 712 | * Success rate : ~0.26% 713 | 714 | [stats](/supplementals/stats/stats-IDPS-Alert-and-Deny-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.csv) 715 | ![Metrics](/supplementals/stats/metrics-IDPS-Alert-and-Deny-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png) 716 | ![Latency](/supplementals/stats/latency-IDPS-Alert-and-Deny-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png) 717 | ### IDPS-Alert-and-Deny-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min 718 | 719 | Variables: 720 | ``` 721 | # terraform.tfvars: 722 | idps = "Deny" 723 | testtype = "vegeta" 724 | testduration = 5 725 | testprotocol = "https" 726 | 727 | 728 | ./manual_vmss_scaling.sh 120 # az cli scaling is much faster than with terraform 729 | ``` 730 | 731 | * Throughput : ~9.2 Gbps 732 | * Latency mean : 29750.68487 ( include server processing delay ) 733 | * Success rate : ~0.23% 734 | 735 | [stats](/supplementals/stats/stats-IDPS-Alert-and-Deny-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.csv) 736 | ![Metrics](/supplementals/stats/metrics-IDPS-Alert-and-Deny-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png) 737 | ![Latency](/supplementals/stats/latency-IDPS-Alert-and-Deny-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png) 738 | ### IDPS-Alert-and-Deny-D4-v4-iperf3-64-30-instances-5-min 739 | 740 | Variables: 741 | ``` 742 | # terraform.tfvars: 743 | idps = "Deny" 744 | testtype = "iperf3" 745 | testduration = 5 746 | testprotocol = "" 747 | 748 | 749 | ./manual_vmss_scaling.sh 30 # az cli scaling is much faster than with terraform 750 | ``` 751 | 752 | * Throughput : ~8.2Gbps 753 | * Latency mean : 2587547.748 754 | 755 | [stats](/supplementals/stats/stats-IDPS-Alert-and-Deny-D4-v4-iperf3-64-30-instances-5-min.csv) 756 | ![Metrics](/supplementals/stats/metrics-IDPS-Alert-and-Deny-D4-v4-iperf3-64-30-instances-5-min.png) 757 | ![Latency](/supplementals/stats/latency-IDPS-Alert-and-Deny-D4-v4-iperf3-64-30-instances-5-min.png) 758 | 759 | 760 | ## Notes and Links 761 | 762 | Nginx tuning https://www.nginx.com/blog/tuning-nginx/ -------------------------------------------------------------------------------- /infrastructure/aks/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | ENV TZ=America/New_York 3 | RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 4 | RUN apt-get update --fix-missing -y 5 | RUN apt-get upgrade -y 6 | RUN apt-get install build-essential -y 7 | RUN apt-get install apt-utils -y 8 | RUN apt-get install software-properties-common -y 9 | #RUN add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu groovy universe" 10 | RUN apt-get update -y 11 | RUN apt-get install net-tools -y 12 | RUN apt-get install iputils-ping -y 13 | RUN apt-get install inetutils-traceroute -y 14 | RUN apt-get install iproute2 -y 15 | RUN apt-get install iperf3 -y 16 | RUN apt-get install ntp -y 17 | RUN apt-get install curl -y 18 | RUN apt-get install vim -y 19 | #RUN timedatectl set-timezone America/New_York 20 | RUN apt-get install jq -y 21 | RUN apt-get install wget 22 | RUN apt-get install python3-pip -y 23 | RUN apt-get install unzip -y 24 | RUN apt-get install less -y 25 | RUN apt-get install nginx -y 26 | # Python Dependencies 27 | RUN pip3 install --upgrade pip 28 | RUN pip3 install azure.servicebus 29 | RUN pip3 install setuptools_rust 30 | RUN pip3 install cryptography 31 | RUN pip3 install azure.cosmosdb.table 32 | RUN wget https://github.com/tsenart/vegeta/releases/download/v12.8.4/vegeta_12.8.4_linux_amd64.tar.gz 33 | RUN tar xvzf vegeta_12.8.4_linux_amd64.tar.gz 34 | RUN cp vegeta /usr/bin 35 | # 36 | # Disabling default logging for nginx 37 | RUN sed -i s/access_log/"# access_log"/ /etc/nginx/nginx.conf 38 | RUN sed -i s/error_log/"# error_log"/ /etc/nginx/nginx.conf 39 | # 40 | # enabling ssl for nginx 41 | RUN openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/ssl-cert-snakeoil.key -out /etc/ssl/certs/ssl-cert-snakeoil.pem -subj "/C=US/CN=fwtest" 42 | # 43 | RUN sed -i "s/\# include snippets\/snakeoil.conf;/ include snippets\/snakeoil.conf;/" /etc/nginx/sites-available/default 44 | RUN sed -i "s/# listen 443/ listen 443/" /etc/nginx/sites-available/default 45 | # 46 | # to test : curl -k https://x.x.x.x 47 | # 48 | # generate random file 49 | RUN mv /var/www/html/index.nginx-debian.html /var/www/html/index.nginx-debian.html.bac 50 | RUN dd if=/dev/urandom bs=1024 count=1 | base64 > /var/www/html/index.nginx-debian.html # for 1M count = 1000 51 | # 52 | -------------------------------------------------------------------------------- /infrastructure/aks/lb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | annotations: 5 | service.beta.kubernetes.io/azure-load-balancer-internal: "true" 6 | labels: 7 | app: nginx 8 | name: nginx 9 | namespace: default 10 | spec: 11 | externalTrafficPolicy: Cluster 12 | ipFamilies: 13 | - IPv4 14 | ipFamilyPolicy: SingleStack 15 | ports: 16 | - name: "80" 17 | port: 80 18 | protocol: TCP 19 | targetPort: 80 20 | - name: "443" 21 | port: 443 22 | protocol: TCP 23 | targetPort: 443 24 | selector: 25 | app: nginx 26 | type: LoadBalancer 27 | status: 28 | loadBalancer: {} 29 | -------------------------------------------------------------------------------- /infrastructure/aks/setup_aks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export rg=`terraform output resource_group_name | tr -d '"'` 4 | export subnet1=`terraform output subnet1_id | tr -d "\""` 5 | export subnet2=`terraform output subnet2_id | tr -d "\""` 6 | 7 | 8 | export vnet1_id=`terraform output vnet1_id | tr -d "\""` 9 | export vnet2_id=`terraform output vnet2_id | tr -d "\""` 10 | 11 | 12 | # ----------------------------------------- 13 | 14 | cd aks/docker 15 | 16 | az acr create -g $rg -n acrcopernic --sku Standard --admin-enabled 17 | az acr update -n acrcopernic --anonymous-pull-enabled 18 | 19 | az acr login --name acrcopernic 20 | 21 | docker build --network=host -t andrew/universalcs . 22 | 23 | docker tag andrew/universalcs acrcopernic.azurecr.io/universalcs 24 | docker push acrcopernic.azurecr.io/universalcs 25 | 26 | az acr repository show -n acrcopernic --image universalcs 27 | 28 | # ----------------------------------------- 29 | 30 | az aks create --resource-group "${rg}" --name aks1 --node-count 3 --enable-addons monitoring \ 31 | --generate-ssh-keys --vnet-subnet-id "${subnet1}" --service-cidr 172.16.0.0/24 --dns-service-ip 172.16.0.10 \ 32 | --network-plugin azure --attach-acr acrcopernic --enable-cluster-autoscaler --min-count 1 --max-count 1000 --node-vm-size Standard_D4_v4 33 | 34 | # az aks delete --name aks1 --resource-group "${rg}" 35 | 36 | az aks get-credentials --name aks1 --resource-group $rg 37 | 38 | kubectl config get-contexts 39 | kubectl config use-context aks1 40 | 41 | # ----------------------------------------- 42 | 43 | az aks create --resource-group "${rg}" --name aks2 --node-count 3 --enable-addons monitoring \ 44 | --generate-ssh-keys --vnet-subnet-id "${subnet2}" --service-cidr 172.16.0.0/24 --dns-service-ip 172.16.0.10 \ 45 | --network-plugin azure --attach-acr acrcopernic --enable-cluster-autoscaler --min-count 1 --max-count 1000 --node-vm-size Standard_D4_v4 46 | 47 | # az aks delete --name aks2 --resource-group "${rg}" 48 | 49 | az aks get-credentials --name aks2 --resource-group $rg 50 | kubectl config get-contexts 51 | kubectl config use-context aks2 52 | 53 | # ------------------------------------------ 54 | 55 | # aks_managed_id=$(az aks show --name aks2 --resource-group $rg --query identity.principalId -o tsv) 56 | # az role assignment create --assignee $aks_managed_id --role "Contributor" --scope $vnet2_id 57 | 58 | # 59 | kubectl config use-context aks2 60 | 61 | kubectl create deployment nginx --image=acrcopernic.azurecr.io/universalcs:latest --replicas=1 -- bash -c "nginx ; sleep infinity" 62 | 63 | kubectl apply -f lb.yaml 64 | 65 | kubectl get svc -o wide 66 | 67 | # ------ 68 | 69 | kubectl config use-context aks1 70 | 71 | kubectl create deployment universalcs --image=acrcopernic.azurecr.io/universalcs:latest \ 72 | cas> --replicas=1 -- bash -c "while true; do echo GET http://10.1.0.5 | vegeta attack -duration=600s -insecure -rate=50 | vegeta report; done" 73 | 74 | # 75 | kubectl config use-context aks2 76 | kubectl scale deploy/nginx --replicas=1000 77 | 78 | # 79 | kubectl config use-context aks1 80 | kubectl scale deploy/universalcs --replicas=1000 -------------------------------------------------------------------------------- /infrastructure/azfirewall.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_public_ip" "azfwpip" { 2 | name = "azfw${random_id.id.hex}" 3 | domain_name_label = "azfw${random_id.id.hex}" 4 | location = var.location 5 | resource_group_name = var.resource_group_name 6 | allocation_method = "Static" 7 | sku = "Standard" 8 | } 9 | 10 | resource "azurerm_firewall" "azfw" { 11 | 12 | dns_servers = null 13 | 14 | private_ip_ranges = null 15 | 16 | name = "AZFWP" 17 | location = var.location 18 | resource_group_name = var.resource_group_name 19 | 20 | sku_tier = "Premium" 21 | 22 | zones = ["1", "2", "3"] 23 | 24 | firewall_policy_id = azurerm_firewall_policy.azfw-policy.id 25 | 26 | ip_configuration { 27 | name = "AZFWPIP" 28 | subnet_id = module.vnet["AZFPVNET"].vnet_subnets[1] 29 | public_ip_address_id = azurerm_public_ip.azfwpip.id 30 | } 31 | 32 | tags = null 33 | } 34 | 35 | resource "azurerm_firewall_policy" "azfw-policy" { 36 | name = "azfw-policy" 37 | 38 | sku = "Premium" 39 | 40 | resource_group_name = var.resource_group_name 41 | location = var.location 42 | 43 | intrusion_detection { 44 | #mode = "Alert" 45 | #mode = "Deny" 46 | #mode = "Off" 47 | mode = var.idps 48 | } 49 | 50 | } 51 | 52 | # Firewall Policy Rules 53 | resource "azurerm_firewall_policy_rule_collection_group" "azfw-policy" { 54 | name = "azfw-policy" 55 | firewall_policy_id = azurerm_firewall_policy.azfw-policy.id 56 | priority = 100 57 | 58 | network_rule_collection { 59 | name = "network_rules1" 60 | priority = 200 61 | action = "Allow" 62 | rule { 63 | name = "iperf3" 64 | protocols = ["Any"] 65 | source_addresses = ["*"] 66 | destination_addresses = ["*"] 67 | destination_ports = ["5201-7201"] 68 | } 69 | rule { 70 | name = "http" 71 | protocols = ["Any"] 72 | source_addresses = ["*"] 73 | destination_addresses = ["*"] 74 | destination_ports = ["80"] 75 | } 76 | rule { 77 | name = "https" 78 | protocols = ["Any"] 79 | source_addresses = ["*"] 80 | destination_addresses = ["*"] 81 | destination_ports = ["443"] 82 | } 83 | rule { 84 | name = "ntttcp5000" 85 | protocols = ["Any"] 86 | source_addresses = ["*"] 87 | destination_addresses = ["*"] 88 | destination_ports = ["5000-6000"] 89 | } 90 | rule { 91 | name = "ethr9999" 92 | protocols = ["Any"] 93 | source_addresses = ["*"] 94 | destination_addresses = ["*"] 95 | destination_ports = ["9999"] 96 | } 97 | rule { 98 | name = "vxlan" 99 | protocols = ["Any"] 100 | source_addresses = ["*"] 101 | destination_addresses = ["*"] 102 | destination_ports = ["4789"] 103 | } 104 | rule { 105 | name = "icmp" 106 | protocols = ["ICMP"] 107 | source_addresses = ["*"] 108 | destination_addresses = ["*"] 109 | destination_ports = ["*"] 110 | } 111 | } 112 | # 113 | # application_rule_collection { 114 | # name = "app_rule_collection1" 115 | # priority = 300 116 | # action = "Deny" 117 | # rule { 118 | # name = "app_rule_collection1_rule1" 119 | # protocols { 120 | # type = "Http" 121 | # port = 80 122 | # } 123 | # protocols { 124 | # type = "Https" 125 | # port = 443 126 | # } 127 | # source_addresses = ["*"] 128 | # destination_fqdns = ["*"] 129 | # } 130 | # } 131 | 132 | 133 | 134 | } -------------------------------------------------------------------------------- /infrastructure/data.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | kvname = "cs-keystore-pm" 3 | } 4 | 5 | data "azurerm_key_vault_secret" "keyvault-username" { 6 | name = "adminusername" 7 | key_vault_id = "/subscriptions/${data.azurerm_subscription.current.subscription_id}/resourceGroups/${var.resource_group_name}/providers/Microsoft.KeyVault/vaults/${local.kvname}" 8 | } 9 | 10 | data "azurerm_key_vault_secret" "keyvault-password" { 11 | name = "adminpassword" 12 | key_vault_id = "/subscriptions/${data.azurerm_subscription.current.subscription_id}/resourceGroups/${var.resource_group_name}/providers/Microsoft.KeyVault/vaults/${local.kvname}" 13 | } 14 | 15 | data "azurerm_ssh_public_key" "sshkey" { 16 | name = "desktop" 17 | resource_group_name = var.resource_group_name 18 | } 19 | 20 | data "azurerm_key_vault_secret" "sharedkey" { 21 | name = "sharedkey" 22 | key_vault_id = "/subscriptions/${data.azurerm_subscription.current.subscription_id}/resourceGroups/${var.resource_group_name}/providers/Microsoft.KeyVault/vaults/${local.kvname}" 23 | } 24 | 25 | data "azurerm_subscription" "current" { 26 | } 27 | 28 | data "external" "my_ip" { 29 | program = [ 30 | "/bin/bash", "scripts/my_ip.sh" 31 | ] 32 | } -------------------------------------------------------------------------------- /infrastructure/entropy.tf: -------------------------------------------------------------------------------- 1 | resource "random_id" "id" { byte_length = 8 } -------------------------------------------------------------------------------- /infrastructure/linux_vm.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | vm_loc = { "spoke1" = var.vnets["spoke1"], 3 | "spoke2" = var.vnets["spoke2"], 4 | "AZFPVNET" = var.vnets["AZFPVNET"] 5 | } 6 | } 7 | 8 | resource "azurerm_network_interface" "nic" { 9 | 10 | for_each = local.vm_loc 11 | #for_each = {} 12 | #for_each = var.vnets 13 | 14 | name = "${each.key}-nic" 15 | location = var.location 16 | resource_group_name = var.resource_group_name 17 | 18 | enable_accelerated_networking = true 19 | 20 | ip_configuration { 21 | name = "internal" 22 | subnet_id = module.vnet[each.key].vnet_subnets[0] 23 | private_ip_address_allocation = "Dynamic" 24 | public_ip_address_id = azurerm_public_ip.pip[each.key].id 25 | } 26 | } 27 | 28 | resource "azurerm_public_ip" "pip" { 29 | 30 | for_each = local.vm_loc 31 | #for_each = {} 32 | #for_each = var.vnets 33 | 34 | name = "${each.key}-pip" 35 | location = var.location 36 | resource_group_name = var.resource_group_name 37 | allocation_method = "Static" 38 | sku = "Standard" 39 | 40 | tags = var.tags 41 | } 42 | 43 | resource "azurerm_storage_account" "boot_diagnostic" { 44 | 45 | for_each = local.vm_loc 46 | #for_each = var.vnets 47 | 48 | name = lower("${random_id.id.hex}${each.key}") 49 | resource_group_name = var.resource_group_name 50 | location = var.location 51 | 52 | account_tier = "Standard" 53 | account_replication_type = "GRS" 54 | 55 | tags = var.tags 56 | } 57 | resource "azurerm_linux_virtual_machine" "vm" { 58 | 59 | for_each = local.vm_loc 60 | 61 | name = "${each.key}-vm" 62 | resource_group_name = var.resource_group_name 63 | location = var.location 64 | size = "Standard_F2" # "Standard_DS5_v2" #"Standard_D48_v3" , "Standard_F2" , Standard_DS4_v2 , Standard_DS5_v2 , Standard_DS4_v2 65 | admin_username = data.azurerm_key_vault_secret.keyvault-username.value 66 | admin_password = data.azurerm_key_vault_secret.keyvault-password.value 67 | 68 | disable_password_authentication = false 69 | 70 | network_interface_ids = [ 71 | azurerm_network_interface.nic[each.key].id, 72 | ] 73 | 74 | admin_ssh_key { 75 | username = data.azurerm_key_vault_secret.keyvault-username.value 76 | public_key = data.azurerm_ssh_public_key.sshkey.public_key 77 | } 78 | 79 | boot_diagnostics { 80 | storage_account_uri = azurerm_storage_account.boot_diagnostic[each.key].primary_blob_endpoint 81 | } 82 | 83 | os_disk { 84 | caching = "ReadWrite" 85 | storage_account_type = "Standard_LRS" 86 | } 87 | 88 | provisioner "local-exec" { 89 | command = "scp -o StrictHostKeyChecking=no /home/andrew/.ssh/id_rsa azureadmin@${azurerm_public_ip.pip[each.key].ip_address}:.ssh/" 90 | } 91 | 92 | source_image_reference { 93 | publisher = "Canonical" 94 | offer = "UbuntuServer" 95 | sku = "18.04-LTS" 96 | version = "latest" 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /infrastructure/load_balancer_outbound.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_public_ip_prefix" "pfpip" { 2 | name = "pfpip" 3 | resource_group_name = var.resource_group_name 4 | location = var.location 5 | prefix_length = 30 6 | sku = "Standard" 7 | } 8 | 9 | 10 | resource "azurerm_lb" "oslb" { 11 | name = "OSLB" 12 | location = var.location 13 | resource_group_name = var.resource_group_name 14 | sku = "Standard" 15 | 16 | frontend_ip_configuration { 17 | name = "pfpip" 18 | #public_ip_address_id = azurerm_public_ip.pfpip.id 19 | public_ip_prefix_id = azurerm_public_ip_prefix.pfpip.id 20 | } 21 | } 22 | 23 | resource "azurerm_lb_backend_address_pool" "oslbpool" { 24 | loadbalancer_id = azurerm_lb.oslb.id 25 | name = "OVMSSAP" 26 | } 27 | 28 | resource "azurerm_lb_outbound_rule" "orule" { 29 | resource_group_name = var.resource_group_name 30 | 31 | allocated_outbound_ports = 256 32 | 33 | loadbalancer_id = azurerm_lb.oslb.id 34 | name = "OutboundRule" 35 | protocol = "All" 36 | backend_address_pool_id = azurerm_lb_backend_address_pool.oslbpool.id 37 | 38 | 39 | frontend_ip_configuration { 40 | name = "pfpip" 41 | } 42 | } -------------------------------------------------------------------------------- /infrastructure/load_balancer_server.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_lb" "slb" { 2 | name = "SLB" 3 | location = var.location 4 | resource_group_name = var.resource_group_name 5 | sku = "Standard" 6 | 7 | frontend_ip_configuration { 8 | name = "FE" 9 | availability_zone = "Zone-Redundant" 10 | subnet_id = module.vnet["spoke2"].vnet_subnets[0] 11 | private_ip_address = cidrhost(var.vnets["spoke2"].subnet_prefixes[0], 65534) 12 | private_ip_address_allocation = "Static" 13 | private_ip_address_version = "IPv4" 14 | } 15 | } 16 | 17 | resource "azurerm_lb_backend_address_pool" "slbpool" { 18 | loadbalancer_id = azurerm_lb.slb.id 19 | name = "VMSSAP" 20 | } 21 | 22 | resource "azurerm_lb_probe" "http" { 23 | resource_group_name = var.resource_group_name 24 | loadbalancer_id = azurerm_lb.slb.id 25 | name = "port80" 26 | port = 80 27 | } 28 | 29 | resource "azurerm_lb_probe" "https" { 30 | resource_group_name = var.resource_group_name 31 | loadbalancer_id = azurerm_lb.slb.id 32 | name = "port443" 33 | port = 443 34 | } 35 | 36 | resource "azurerm_lb_rule" "http" { 37 | resource_group_name = var.resource_group_name 38 | loadbalancer_id = azurerm_lb.slb.id 39 | name = "http" 40 | protocol = "Tcp" 41 | frontend_port = 80 42 | backend_port = 80 43 | frontend_ip_configuration_name = "FE" 44 | probe_id = azurerm_lb_probe.http.id 45 | backend_address_pool_ids = [azurerm_lb_backend_address_pool.slbpool.id] 46 | } 47 | 48 | resource "azurerm_lb_rule" "https" { 49 | resource_group_name = var.resource_group_name 50 | loadbalancer_id = azurerm_lb.slb.id 51 | name = "https" 52 | protocol = "Tcp" 53 | frontend_port = 443 54 | backend_port = 443 55 | frontend_ip_configuration_name = "FE" 56 | probe_id = azurerm_lb_probe.https.id 57 | backend_address_pool_ids = [azurerm_lb_backend_address_pool.slbpool.id] 58 | } -------------------------------------------------------------------------------- /infrastructure/outputs.tf: -------------------------------------------------------------------------------- 1 | output "resource_group_name" { 2 | value = var.resource_group_name 3 | } 4 | 5 | output "location" { 6 | value = var.location 7 | } 8 | 9 | output "Spoke1_Linux_public_ip" { 10 | value = azurerm_public_ip.pip["spoke1"].ip_address 11 | } 12 | 13 | output "Spoke2_Linux_public_ip" { 14 | value = azurerm_public_ip.pip["spoke2"].ip_address 15 | } 16 | 17 | output "AZFW_Linux_public_ip" { 18 | value = azurerm_public_ip.pip["AZFPVNET"].ip_address 19 | } 20 | 21 | output "rg" { 22 | value = var.resource_group_name 23 | } 24 | 25 | output "my_ip" { 26 | value = data.external.my_ip.result.my_ip 27 | } 28 | 29 | output "clients" { 30 | value = azurerm_linux_virtual_machine_scale_set.clients["spoke1"].name 31 | } 32 | 33 | 34 | output "servers" { 35 | value = azurerm_linux_virtual_machine_scale_set.servers["spoke2"].name 36 | } 37 | 38 | output "subnet1_id" { 39 | value = module.vnet["spoke1"].vnet_subnets[0] 40 | } 41 | 42 | 43 | output "subnet2_id" { 44 | value = module.vnet["spoke2"].vnet_subnets[0] 45 | } 46 | 47 | 48 | output "vnet1_id" { 49 | value = module.vnet["spoke1"].vnet_id 50 | } 51 | 52 | 53 | output "vnet2_id" { 54 | value = module.vnet["spoke2"].vnet_id 55 | } -------------------------------------------------------------------------------- /infrastructure/route_table.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_route_table" "RT" { 2 | name = "rt" 3 | location = var.location 4 | resource_group_name = var.resource_group_name 5 | disable_bgp_route_propagation = true 6 | 7 | depends_on = [ 8 | azurerm_resource_group.rg 9 | ] 10 | 11 | route { 12 | name = "default" 13 | address_prefix = "10.0.0.0/8" 14 | next_hop_type = "VirtualAppliance" 15 | next_hop_in_ip_address = azurerm_firewall.azfw.ip_configuration[0].private_ip_address 16 | } 17 | 18 | } 19 | 20 | resource "azurerm_subnet_route_table_association" "RT" { 21 | for_each = var.vnets 22 | 23 | subnet_id = module.vnet[each.key].vnet_subnets[0] 24 | route_table_id = azurerm_route_table.RT.id 25 | } 26 | -------------------------------------------------------------------------------- /infrastructure/scripts/client.txt: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Dependencies 4 | apt-get update -y 5 | apt-get upgrade -y 6 | #apt-get install iproute2 -y 7 | #apt-get install wrk -y 8 | apt-get install iperf3 -y 9 | apt-get install ntp -y 10 | apt-get install jq -y 11 | apt-get install python3-pip -y 12 | apt-get install unzip -y 13 | 14 | # Python Dependencies 15 | pip3 install --upgrade pip 16 | pip3 install azure.servicebus 17 | pip3 install setuptools_rust 18 | pip3 install cryptography 19 | pip3 install azure.cosmosdb.table 20 | 21 | # Increasing number of descriptors 22 | ulimit -n 10000 23 | 24 | # Adjusting the timezone 25 | timedatectl set-timezone America/New_York 26 | 27 | # ethr installation (uncomment if tool is needed) 28 | # cd ~ 29 | # wget https://github.com/microsoft/ethr/releases/download/v1.0.0/ethr_linux.zip 30 | # unzip ethr_linux.zip 31 | # chmod +x ethr 32 | # cp ethr /usr/bin 33 | 34 | # wrk testing suite (uncomment if tool is needed) 35 | # cd ~ 36 | # git clone https://github.com/wg/wrk 37 | # cd wrk 38 | # make 39 | # chmod +x wrk 40 | # cp wrk /usr/bin/ 41 | 42 | # ntttcp (uncomment if tool is needed) 43 | # cd ~ 44 | # git clone https://github.com/Microsoft/ntttcp-for-linux 45 | # cd ntttcp-for-linux/src 46 | # make 47 | # make install 48 | 49 | cd ~ 50 | 51 | # vegeta 52 | cd ~ 53 | wget https://github.com/tsenart/vegeta/releases/download/v12.8.4/vegeta_12.8.4_linux_amd64.tar.gz 54 | tar xvzf vegeta_12.8.4_linux_amd64.tar.gz 55 | cp vegeta /usr/bin 56 | 57 | # iperf3 test suite 58 | cat << EOF>client.py 59 | from azure.servicebus import ServiceBusClient, ServiceBusMessage 60 | from azure.cosmosdb.table.tableservice import TableService 61 | from azure.cosmosdb.table.models import Entity 62 | 63 | import datetime 64 | import time 65 | import socket 66 | import os 67 | import random 68 | 69 | def get_sourceip(): 70 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 71 | try: 72 | # doesn't even have to be reachable 73 | s.connect(('1.1.1.1', 1)) 74 | IP = s.getsockname()[0] 75 | except Exception: 76 | IP = '127.0.0.1' 77 | finally: 78 | s.close() 79 | return IP 80 | 81 | def get_serverip(): 82 | # create a Service Bus client using the connection string 83 | servicebus_client = ServiceBusClient.from_connection_string(conn_str='${CONNECTION_STR}', logging_enable=True) 84 | 85 | with servicebus_client: 86 | msg = None 87 | while not msg: 88 | receiver = servicebus_client.get_queue_receiver(queue_name='${QUEUE_NAME}', max_wait_time=5) 89 | 90 | with receiver: 91 | for msg in receiver: 92 | receiver.complete_message(msg) 93 | return (str(msg)) 94 | 95 | def measure_throughput(ip,server_port,duration,streams=64): 96 | cport=int(random.random()*(65535-49152) + 49152) 97 | o=os.popen(f"iperf3 -c {ip} -t {duration} --cport {cport} --port {server_port} -P{streams} -J | jq \".end.sum_sent.bits_per_second, .end.sum_received.bits_per_second, .end.streams[0].sender.max_rtt, .end.streams[0].sender.min_rtt, .end.streams[0].sender.mean_rtt\" ").read() 98 | 99 | return (o.split("\n")) 100 | 101 | 102 | def measure_http(ip,duration,rate=50,protocol="http"): 103 | filter = '.latencies.total, .latencies.mean, .latencies.max, .latencies.min, .throughput, .success' 104 | 105 | o=os.popen(f"echo \"GET {protocol}://{ip}\" | vegeta attack -duration={duration}s -insecure -rate={rate} | vegeta report -type=json | jq \"{filter}\" ").read() 106 | 107 | return (o.split("\n")) 108 | 109 | scalefactor_vegeta = 1000000 110 | scalefactor_iperf3 = 1 111 | 112 | source_ip = get_sourceip() 113 | 114 | server_ip,server_port = get_serverip().split() 115 | #server_ip = "10.1.255.254" # if load balancer is in use 116 | 117 | duration = ${TESTDURATION} # in minutes 118 | 119 | streams = ${TESTIPERF3FLOWS} # iperf3 specific 120 | 121 | rate = 50 # # vegeta specific 122 | protocol = "${TESTPROTOCOL}" # vegeta specific 123 | 124 | test_type = "${TESTTYPE}" # iperf3 , vegeta 125 | 126 | 127 | # ------------------------------ 128 | 129 | 130 | table_service = TableService(account_name='${STORAGEACCOUNT}', account_key='${STORAGEACCOUNTKEY}') 131 | 132 | while True: 133 | if int(datetime.datetime.now().strftime("%S")) == 0 and int(datetime.datetime.now().strftime("%M"))%duration == 0: 134 | n = datetime.datetime.now().strftime("%m %d %Y %H %M %S") 135 | 136 | # -------------------------------- 137 | 138 | entity = Entity() 139 | entity.PartitionKey = source_ip 140 | entity.RowKey = n 141 | 142 | if test_type == "iperf3": 143 | throughput = measure_throughput(server_ip,server_port,(duration-1)*60,streams) 144 | entity.test_type = "iperf3" 145 | entity.sent_throughput = throughput[0] 146 | entity.received_throughput = throughput[1] 147 | entity.max_rtt = int(throughput[2])/scalefactor_iperf3 148 | entity.min_rtt = int(throughput[3])/scalefactor_iperf3 149 | entity.mean_rtt = int(throughput[4])/scalefactor_iperf3 150 | entity.server = server_ip + ":" + server_port 151 | entity.duration = duration 152 | elif test_type == "vegeta": 153 | throughput = measure_http(server_ip,(duration-1)*60,rate=rate,protocol=protocol) 154 | entity.test_type = "vegeta" 155 | entity.protocol = protocol 156 | entity.latency_total = int(throughput[0])/scalefactor_vegeta 157 | entity.latency_mean = int(throughput[1])/scalefactor_vegeta 158 | entity.latency_max = int(throughput[2])/scalefactor_vegeta 159 | entity.latency_min = int(throughput[3])/scalefactor_vegeta 160 | entity.throughput = throughput[4] 161 | entity.success = throughput[5] 162 | 163 | entity.server = server_ip 164 | #print("ready to insert",entity) 165 | table_service.insert_entity('stats', entity) 166 | #print("inserted",entity) 167 | 168 | EOF 169 | 170 | # Uncomment for traffic generator 171 | python3 client.py -------------------------------------------------------------------------------- /infrastructure/scripts/manual_storage_account_cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | storage_account_name="ttresult" 4 | table_name="stats" 5 | 6 | rg=`terraform output resource_group_name | tr -d '"'` 7 | location=`terraform output location | tr -d '"'` 8 | 9 | echo "Cleaning up storage account $storage_account_name at $rg / $location" 10 | 11 | 12 | az storage account delete \ 13 | --name $storage_account_name \ 14 | --resource-group $rg \ 15 | --yes 16 | 17 | az storage account create \ 18 | --name $storage_account_name \ 19 | --resource-group $rg \ 20 | --location $location \ 21 | --sku Standard_GRS \ 22 | --kind StorageV2 23 | 24 | az storage table create --name $table_name \ 25 | --account-name $storage_account_name 26 | -------------------------------------------------------------------------------- /infrastructure/scripts/manual_vmss_scaling.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z "$1" ]; then 4 | echo "Usage: $0 number" 5 | exit 1 6 | fi 7 | 8 | number=$1 9 | 10 | echo "Started VMSS scale at `date`" 11 | 12 | az vmss scale --new-capacity $number --name `terraform output servers | tr -d '"'` --resource-group `terraform output resource_group_name | tr -d '"'` --no-wait 13 | 14 | az vmss scale --new-capacity $number --name `terraform output clients | tr -d '"'` --resource-group `terraform output resource_group_name | tr -d '"'` --no-wait 15 | -------------------------------------------------------------------------------- /infrastructure/scripts/my_ip.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | function error_exit() { 3 | echo "$1" 1>&2 4 | exit 1 5 | } 6 | 7 | function my_ip() { 8 | ip=$(curl -s https://api.my-ip.io/ip.json | jq "{ my_ip: .ip }") 9 | echo $ip 10 | } 11 | 12 | my_ip -------------------------------------------------------------------------------- /infrastructure/scripts/server.txt: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # System Dependencies 4 | apt-get update -y 5 | apt-get upgrade -y 6 | apt-get install iproute2 -y 7 | #apt-get install wrk -y 8 | apt-get install iperf3 -y 9 | apt-get install ntp -y 10 | apt-get install nginx -y 11 | apt-get install unzip -y 12 | 13 | # Increasing number of descriptors 14 | ulimit -n 10000 15 | 16 | # Disabling default logging for nginx 17 | sed -i s/access_log/"# access_log"/ /etc/nginx/nginx.conf 18 | sed -i s/error_log/"# error_log"/ /etc/nginx/nginx.conf 19 | 20 | # enabling ssl for nginx 21 | openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/ssl-cert-snakeoil.key -out /etc/ssl/certs/ssl-cert-snakeoil.pem -subj "/C=US/CN=fwtest" 22 | 23 | sed -i "s/\# include snippets\/snakeoil.conf;/ include snippets\/snakeoil.conf;/" /etc/nginx/sites-available/default 24 | sed -i "s/# listen 443/ listen 443/" /etc/nginx/sites-available/default 25 | 26 | # to test : curl -k https://x.x.x.x 27 | 28 | # generate random file 29 | mv /var/www/html/index.nginx-debian.html /var/www/html/index.nginx-debian.html.bac 30 | dd if=/dev/urandom bs=1024 count=1000 | base64 > /var/www/html/index.nginx-debian.html # for 1M count = 1000 31 | 32 | service nginx restart 33 | 34 | # python Dependencies 35 | sudo apt install python3-pip -y 36 | pip3 install azure.servicebus 37 | 38 | # ethr installation 39 | # wget https://github.com/microsoft/ethr/releases/download/v1.0.0/ethr_linux.zip 40 | # unzip ethr_linux.zip 41 | # chmod +x ethr 42 | # cp ethr /usr/bin 43 | 44 | # ntttcp installation 45 | # cd ~ 46 | # git clone https://github.com/Microsoft/ntttcp-for-linux 47 | # cd ntttcp-for-linux/src 48 | # make 49 | # make install 50 | 51 | cd ~ 52 | 53 | # python test suite ( mainly for reporting ip to servicebus) 54 | cat << EOF>server.py 55 | from azure.servicebus import ServiceBusClient, ServiceBusMessage 56 | import socket 57 | import random 58 | import os 59 | 60 | def get_ip(): 61 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 62 | try: 63 | # doesn't even have to be reachable 64 | s.connect(('1.1.1.1', 1)) 65 | IP = s.getsockname()[0] 66 | except Exception: 67 | IP = '127.0.0.1' 68 | finally: 69 | s.close() 70 | return(IP) 71 | 72 | def send_single_message(sender,msgtext): 73 | # create a Service Bus message 74 | 75 | message = ServiceBusMessage( msgtext ) 76 | # send the message to the queue 77 | sender.send_messages(message) 78 | #print("Sent a single message") 79 | 80 | # create a Service Bus client using the connection string 81 | servicebus_client = ServiceBusClient.from_connection_string(conn_str='${CONNECTION_STR}', logging_enable=True) 82 | 83 | with servicebus_client: 84 | # get a Queue Sender object to send messages to the queue 85 | sender = servicebus_client.get_queue_sender(queue_name='${QUEUE_NAME}') 86 | with sender: 87 | server_port = int(random.random()*(7201 - 5201) + 5201) 88 | msgtext = get_ip() + " " + str(server_port) 89 | send_single_message(sender,msgtext) 90 | 91 | #print("Done sending messages") 92 | #print("-----------------------") 93 | 94 | o=os.popen(f"iperf3 -s -D -p {server_port}") 95 | 96 | EOF 97 | 98 | #Uncomment for traffic generator 99 | python3 server.py 100 | 101 | -------------------------------------------------------------------------------- /infrastructure/service_bus.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_servicebus_namespace" "this" { 2 | name = "tt${random_id.id.hex}" 3 | 4 | location = var.location 5 | resource_group_name = var.resource_group_name 6 | 7 | sku = "Standard" 8 | 9 | } 10 | 11 | resource "azurerm_servicebus_queue" "this" { 12 | name = "ipserverip" 13 | resource_group_name = var.resource_group_name 14 | namespace_name = azurerm_servicebus_namespace.this.name 15 | 16 | #auto_delete_on_idle = "PT10M" 17 | default_message_ttl = "PT10M" 18 | 19 | enable_partitioning = true 20 | 21 | dead_lettering_on_message_expiration = true 22 | } 23 | 24 | resource "azurerm_servicebus_namespace_authorization_rule" "this" { 25 | 26 | name = "authorization" 27 | resource_group_name = var.resource_group_name 28 | namespace_name = azurerm_servicebus_namespace.this.name 29 | 30 | listen = true 31 | send = true 32 | manage = false 33 | } -------------------------------------------------------------------------------- /infrastructure/storage_account_table.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_storage_account" "tresult" { 2 | name = "ttresult" 3 | resource_group_name = var.resource_group_name 4 | location = var.location 5 | account_tier = "Standard" 6 | account_replication_type = "GRS" 7 | 8 | shared_access_key_enabled = true 9 | 10 | } 11 | 12 | resource "azurerm_storage_table" "tresult" { 13 | name = "stats" 14 | storage_account_name = azurerm_storage_account.tresult.name 15 | } -------------------------------------------------------------------------------- /infrastructure/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.13.0" 3 | 4 | required_providers { 5 | azurerm = { 6 | source = "hashicorp/azurerm" 7 | version = ">= 2.41.0" 8 | 9 | } 10 | } 11 | backend "local" { 12 | path = "terraform.tfstate" 13 | } 14 | } 15 | 16 | provider "azurerm" { 17 | features {} 18 | } -------------------------------------------------------------------------------- /infrastructure/terraform.tfvars: -------------------------------------------------------------------------------- 1 | location = "centralus" 2 | 3 | resource_group_name = "AZFP" 4 | 5 | tags = { 6 | Terraform = "true" 7 | Environment = "dev" 8 | } 9 | 10 | lock_level = "" 11 | 12 | security_group_name = "nsg" 13 | 14 | vnets = { 15 | "AZFPVNET" = { 16 | address_space = ["10.0.0.0/16"] 17 | subnet_names = ["default", "AzureFirewallSubnet"] 18 | subnet_prefixes = ["10.0.1.0/24", "10.0.0.0/24"] 19 | enforce_private_link_endpoint_network_policies = { 20 | default = false # "privateEndpointNetworkPolicies": "Disabled=true Enabled=false", 21 | AzureFirewallSubnet = false # "privateEndpointNetworkPolicies": "Disabled=true Enabled=false" 22 | } 23 | enforce_private_link_service_network_policies = { 24 | default = false # "privateEndpointNetworkPolicies": "Disabled=true Enabled=false", 25 | AzureFirewallSubnet = false # "privateEndpointNetworkPolicies": "Disabled=true Enabled=false" 26 | } 27 | }, 28 | "spoke1" = { 29 | address_space = ["10.2.0.0/16"] 30 | subnet_names = ["default"] 31 | subnet_prefixes = ["10.2.0.0/16"] 32 | enforce_private_link_endpoint_network_policies = { 33 | default = true # "privateEndpointNetworkPolicies": "Disabled=true Enabled=false" 34 | } 35 | enforce_private_link_service_network_policies = { 36 | default = true # "privateLinkServiceNetworkPolicies": "Disabled=true Enabled=false" 37 | } 38 | }, 39 | "spoke2" = { 40 | address_space = ["10.1.0.0/16"] 41 | subnet_names = ["default"] 42 | subnet_prefixes = ["10.1.0.0/16"] 43 | enforce_private_link_endpoint_network_policies = { 44 | default = true # "privateEndpointNetworkPolicies": "Disabled=true Enabled=false", 45 | } 46 | enforce_private_link_service_network_policies = { 47 | default = true # "privateLinkServiceNetworkPolicies": "Disabled=true Enabled=false", 48 | } 49 | } 50 | } 51 | 52 | vmss_size = "Standard_D4_v4" # " will try Standard_D2_v4 Standard_DS1_v2 Standard_DS1_v2 and F1s" "Standard_D4_v4" #"Standard_D4_v4" #"Standard_DS3_v2" # "Standard_D3_v2" 53 | 54 | vmssscale = 0 55 | 56 | idps = "Deny" # Off, Alert, Deny 57 | 58 | testtype = "vegeta" # iperf3 or vegeta 59 | testduration = "5" 60 | # vegeta specific 61 | testprotocol = "https" # not relevant if testtype=iperf3 62 | # iperf3 specific 63 | testiperf3flows = "64" # not relevant if testtype=vegeta 64 | 65 | -------------------------------------------------------------------------------- /infrastructure/variables.tf: -------------------------------------------------------------------------------- 1 | variable "location" { 2 | description = "The Azure Region where the Resources should exist." 3 | type = string 4 | } 5 | 6 | variable "resource_group_name" { 7 | description = "The Name which should be used for this Resource Group." 8 | type = string 9 | 10 | validation { 11 | condition = length(var.resource_group_name) >= 1 && length(var.resource_group_name) <= 90 && can(regex("^[a-zA-Z0-9-._\\(\\)]+[a-zA-Z0-9-_\\(\\)]$", var.resource_group_name)) 12 | error_message = "Invalid name (check Azure Resource naming restrictions for more info)." 13 | } 14 | } 15 | 16 | variable "tags" { 17 | description = "A mapping of tags which should be assigned to the resources" 18 | type = map(string) 19 | default = {} 20 | } 21 | 22 | variable "lock_level" { 23 | description = "Specifies the Level to be used for this RG Lock. Possible values are Empty (no lock), CanNotDelete and ReadOnly." 24 | type = string 25 | default = "" 26 | } 27 | 28 | variable "security_group_name" { 29 | description = "NSG Name" 30 | type = string 31 | default = "nsg" 32 | } 33 | 34 | variable "vnets" { 35 | description = "List of Vnets names" 36 | type = map(object( 37 | { 38 | address_space = list(string) 39 | subnet_names = list(string) 40 | subnet_prefixes = list(string) 41 | enforce_private_link_endpoint_network_policies = map(any) 42 | enforce_private_link_service_network_policies = map(any) 43 | } 44 | )) 45 | default = {} 46 | } 47 | 48 | variable "vmssscale" { 49 | description = "Number of instances in VMSS" 50 | } 51 | 52 | variable "idps" { 53 | description = "idps mode of the firewall. Allowed values are Alert,Deny,Off" 54 | } 55 | 56 | variable "testtype" { 57 | description = "Test type. Allowed Values are : vegeta, iperf3" 58 | } 59 | 60 | variable "testduration" { 61 | description = "Test duration in minutes" 62 | } 63 | 64 | variable "testprotocol" { 65 | description = "Test protocol. Allowed values are http, https. Ignored if testtype is iperf3" 66 | default = "http" 67 | } 68 | 69 | variable "testiperf3flows" { 70 | description = "iperf3 specific. Number of flows" 71 | default = "64" 72 | } 73 | 74 | variable "vmss_size" { 75 | description = "VMSS VM Size" 76 | default = "Standard_D4_v4" 77 | } -------------------------------------------------------------------------------- /infrastructure/vm_shutdown_schedule.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | resource "azurerm_dev_test_global_vm_shutdown_schedule" "shutdown-linux" { 4 | 5 | #for_each = azurerm_linux_virtual_machine.vm 6 | for_each = {} 7 | 8 | virtual_machine_id = each.value.id 9 | location = var.location 10 | enabled = true 11 | 12 | daily_recurrence_time = "2300" 13 | timezone = "Eastern Standard Time" 14 | 15 | notification_settings { 16 | enabled = false 17 | time_in_minutes = "60" 18 | webhook_url = "https://sample-webhook-url.example.com" 19 | } 20 | } -------------------------------------------------------------------------------- /infrastructure/vmss-clients.tf: -------------------------------------------------------------------------------- 1 | data "template_file" "client" { 2 | 3 | template = file("scripts/client.txt") 4 | vars = { 5 | CONNECTION_STR = azurerm_servicebus_namespace_authorization_rule.this.primary_connection_string 6 | QUEUE_NAME = azurerm_servicebus_queue.this.name 7 | STORAGEACCOUNT = azurerm_storage_account.tresult.name 8 | STORAGEACCOUNTKEY = azurerm_storage_account.tresult.primary_access_key 9 | TESTTYPE = var.testtype 10 | TESTDURATION = var.testduration 11 | TESTPROTOCOL = var.testprotocol 12 | TESTIPERF3FLOWS = var.testiperf3flows 13 | } 14 | } 15 | 16 | resource "azurerm_linux_virtual_machine_scale_set" "clients" { 17 | 18 | for_each = { 19 | "spoke1" = var.vnets["spoke1"] 20 | } 21 | 22 | name = "clients${each.key}" 23 | 24 | location = var.location 25 | resource_group_name = var.resource_group_name 26 | 27 | sku = var.vmss_size 28 | 29 | instances = var.vmssscale 30 | admin_username = data.azurerm_key_vault_secret.keyvault-username.value 31 | 32 | custom_data = base64encode(data.template_file.client.rendered) 33 | 34 | single_placement_group = false 35 | 36 | zone_balance = false 37 | #zones = [1,2,3] 38 | zones = [1] 39 | 40 | admin_ssh_key { 41 | username = data.azurerm_key_vault_secret.keyvault-username.value 42 | public_key = data.azurerm_ssh_public_key.sshkey.public_key 43 | } 44 | 45 | source_image_reference { 46 | publisher = "canonical" 47 | offer = "0001-com-ubuntu-server-focal" 48 | sku = "20_04-lts-gen2" 49 | version = "latest" 50 | } 51 | 52 | os_disk { 53 | storage_account_type = "Standard_LRS" 54 | caching = "ReadWrite" 55 | } 56 | 57 | network_interface { 58 | name = "nic" 59 | primary = true 60 | enable_accelerated_networking = "true" 61 | 62 | ip_configuration { 63 | 64 | name = "internal" 65 | primary = true 66 | subnet_id = module.vnet[each.key].vnet_subnets[0] 67 | } 68 | } 69 | } -------------------------------------------------------------------------------- /infrastructure/vmss-servers.tf: -------------------------------------------------------------------------------- 1 | data "template_file" "server" { 2 | 3 | template = file("scripts/server.txt") 4 | vars = { 5 | CONNECTION_STR = azurerm_servicebus_namespace_authorization_rule.this.primary_connection_string 6 | QUEUE_NAME = azurerm_servicebus_queue.this.name 7 | } 8 | } 9 | 10 | 11 | resource "azurerm_linux_virtual_machine_scale_set" "servers" { 12 | 13 | for_each = { 14 | "spoke2" = var.vnets["spoke2"] 15 | } 16 | 17 | name = "servers${each.key}" 18 | 19 | location = var.location 20 | resource_group_name = var.resource_group_name 21 | 22 | sku = var.vmss_size 23 | 24 | instances = var.vmssscale 25 | admin_username = data.azurerm_key_vault_secret.keyvault-username.value 26 | 27 | upgrade_mode = "Automatic" 28 | single_placement_group = false 29 | 30 | zone_balance = false 31 | #zones = [1,2,3] 32 | zones = [1] 33 | 34 | custom_data = base64encode(data.template_file.server.rendered) 35 | 36 | #health_probe_id = azurerm_lb_probe.http.id 37 | 38 | admin_ssh_key { 39 | username = data.azurerm_key_vault_secret.keyvault-username.value 40 | public_key = data.azurerm_ssh_public_key.sshkey.public_key 41 | } 42 | 43 | source_image_reference { 44 | publisher = "canonical" 45 | offer = "0001-com-ubuntu-server-focal" 46 | sku = "20_04-lts-gen2" 47 | version = "latest" 48 | } 49 | 50 | os_disk { 51 | storage_account_type = "Standard_LRS" 52 | caching = "ReadWrite" 53 | } 54 | 55 | network_interface { 56 | name = "nic" 57 | primary = true 58 | enable_accelerated_networking = "true" 59 | 60 | ip_configuration { 61 | 62 | name = "internal" 63 | primary = true 64 | subnet_id = module.vnet[each.key].vnet_subnets[0] 65 | 66 | #load_balancer_backend_address_pool_ids = [azurerm_lb_backend_address_pool.slbpool.id, azurerm_lb_backend_address_pool.oslbpool.id ] 67 | 68 | } 69 | } 70 | } -------------------------------------------------------------------------------- /infrastructure/vnet-peering.tf: -------------------------------------------------------------------------------- 1 | 2 | module "vnet-peering01" { 3 | source = "../modules/vnet-peering" 4 | 5 | resource_group_name = var.resource_group_name 6 | location = var.location 7 | 8 | peering = [{ 9 | vnet_id = module.vnet["AZFPVNET"].vnet_id 10 | vnet_name = module.vnet["AZFPVNET"].vnet_name 11 | allow_virtual_network_access = true 12 | allow_forwarded_traffic = true 13 | allow_gateway_transit = false 14 | use_remote_gateways = false 15 | }, 16 | { 17 | vnet_id = module.vnet["spoke1"].vnet_id 18 | vnet_name = module.vnet["spoke1"].vnet_name 19 | allow_virtual_network_access = true 20 | allow_forwarded_traffic = true 21 | allow_gateway_transit = false 22 | use_remote_gateways = false 23 | } 24 | ] 25 | 26 | } 27 | 28 | 29 | module "vnet-peering02" { 30 | source = "../modules/vnet-peering" 31 | 32 | resource_group_name = var.resource_group_name 33 | location = var.location 34 | 35 | peering = [{ 36 | vnet_id = module.vnet["AZFPVNET"].vnet_id 37 | vnet_name = module.vnet["AZFPVNET"].vnet_name 38 | allow_virtual_network_access = true 39 | allow_forwarded_traffic = true 40 | allow_gateway_transit = false 41 | use_remote_gateways = false 42 | }, 43 | { 44 | vnet_id = module.vnet["spoke2"].vnet_id 45 | vnet_name = module.vnet["spoke2"].vnet_name 46 | allow_virtual_network_access = true 47 | allow_forwarded_traffic = true 48 | allow_gateway_transit = false 49 | use_remote_gateways = false 50 | } 51 | ] 52 | 53 | } -------------------------------------------------------------------------------- /infrastructure/vnet.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "rg" { 2 | name = var.resource_group_name 3 | location = var.location 4 | tags = var.tags 5 | } 6 | 7 | module "nsg" { 8 | source = "Azure/network-security-group/azurerm" 9 | version = "3.6.0" 10 | 11 | resource_group_name = var.resource_group_name 12 | location = var.location 13 | security_group_name = var.security_group_name 14 | 15 | custom_rules = [ 16 | { 17 | name = "R100" 18 | priority = 100 19 | direction = "Inbound" 20 | access = "Allow" 21 | protocol = "tcp" 22 | source_port_range = "*" 23 | destination_port_range = "22,80,443" 24 | source_address_prefix = data.external.my_ip.result.my_ip 25 | description = "MGMT" 26 | } 27 | ] 28 | 29 | tags = var.tags 30 | 31 | # to overcome an error of non-existent RG 32 | depends_on = [ 33 | azurerm_resource_group.rg 34 | ] 35 | } 36 | 37 | 38 | module "vnet" { 39 | source = "Azure/vnet/azurerm" 40 | version = "2.5.0" 41 | 42 | for_each = var.vnets 43 | 44 | resource_group_name = var.resource_group_name 45 | vnet_name = each.key 46 | 47 | address_space = each.value.address_space 48 | 49 | subnet_names = each.value.subnet_names 50 | subnet_prefixes = each.value.subnet_prefixes 51 | 52 | subnet_enforce_private_link_endpoint_network_policies = each.value.enforce_private_link_endpoint_network_policies 53 | subnet_enforce_private_link_service_network_policies = each.value.enforce_private_link_service_network_policies 54 | 55 | #nsg_ids = { 56 | # "default" = module.nsg.network_security_group_id 57 | #} 58 | 59 | tags = var.tags 60 | vnet_location = var.location 61 | 62 | # to overcome an error of non-existent RG 63 | depends_on = [ 64 | azurerm_resource_group.rg 65 | ] 66 | } 67 | 68 | resource "azurerm_subnet_network_security_group_association" "this" { 69 | 70 | for_each = var.vnets 71 | 72 | subnet_id = module.vnet[each.key].vnet_subnets[0] 73 | network_security_group_id = module.nsg.network_security_group_id 74 | } 75 | -------------------------------------------------------------------------------- /modules/generic_vm/main.tf: -------------------------------------------------------------------------------- 1 | 2 | resource "azurerm_network_interface" "nic" { 3 | 4 | count = length(var.subnet_ids) 5 | 6 | name = "${var.name}-nic-${count.index}" 7 | location = var.location 8 | resource_group_name = var.resource_group_name 9 | 10 | enable_ip_forwarding = var.enable_ip_forwarding 11 | 12 | ip_configuration { 13 | name = "internal" 14 | subnet_id = var.subnet_ids[count.index] 15 | private_ip_address_allocation = "Dynamic" 16 | public_ip_address_id = count.index == 0 ? var.public_ip_address_id : "" 17 | } 18 | } 19 | 20 | resource "azurerm_virtual_machine" "generic_vm" { 21 | name = var.name 22 | resource_group_name = var.resource_group_name 23 | location = var.location 24 | vm_size = var.vm_size 25 | 26 | delete_os_disk_on_termination = true 27 | delete_data_disks_on_termination = true 28 | 29 | primary_network_interface_id = azurerm_network_interface.nic[0].id 30 | network_interface_ids = azurerm_network_interface.nic.*.id 31 | 32 | zones = [ var.zone ] 33 | 34 | os_profile { 35 | computer_name = var.name 36 | admin_username = var.admin_username 37 | admin_password = var.admin_password 38 | } 39 | 40 | os_profile_linux_config { 41 | disable_password_authentication = var.disable_password_authentication 42 | ssh_keys { 43 | path = "/home/${var.admin_username}/.ssh/authorized_keys" 44 | key_data = var.ssh_key_data 45 | } 46 | } 47 | 48 | storage_image_reference { 49 | publisher = var.storage_image_reference.publisher 50 | offer = var.storage_image_reference.offer 51 | sku = var.storage_image_reference.sku 52 | version = var.storage_image_reference.version 53 | } 54 | 55 | plan { 56 | name = var.plan.name 57 | publisher = var.plan.publisher 58 | product = var.plan.product 59 | } 60 | 61 | storage_os_disk { 62 | name = "${var.name}osdisk" 63 | caching = "ReadWrite" 64 | create_option = "FromImage" 65 | managed_disk_type = "Standard_LRS" 66 | } 67 | 68 | 69 | } 70 | 71 | -------------------------------------------------------------------------------- /modules/generic_vm/outputs.tf: -------------------------------------------------------------------------------- 1 | output "azurerm_virtual_machine_name" { 2 | value = azurerm_virtual_machine.generic_vm.name 3 | } 4 | 5 | output "azurerm_virtual_machine_id" { 6 | value = azurerm_virtual_machine.generic_vm.id 7 | } 8 | 9 | output "azurerm_network_interface_names" { 10 | value = azurerm_network_interface.nic.*.name 11 | } 12 | 13 | output "azurerm_network_interface_id" { 14 | value = azurerm_network_interface.nic.*.id 15 | } 16 | 17 | output "azurerm_network_interface_private_ip_address" { 18 | value = azurerm_network_interface.nic.*.private_ip_address 19 | } 20 | -------------------------------------------------------------------------------- /modules/generic_vm/variables.tf: -------------------------------------------------------------------------------- 1 | variable "subnet_ids" { 2 | description = "Subnet IDs where VM will be placed" 3 | type = list(string) 4 | } 5 | 6 | variable "name" { 7 | description = "Name of VM" 8 | type = string 9 | default = "LinuxVM" 10 | } 11 | 12 | variable "resource_group_name" { 13 | description = "Resource Group Name" 14 | type = string 15 | } 16 | 17 | variable "location" { 18 | description = "The location of the vnet to create. Defaults to the location of the resource group." 19 | type = string 20 | default = null 21 | } 22 | 23 | variable "enable_ip_forwarding" { 24 | type = bool 25 | default = false 26 | } 27 | 28 | variable "vm_size" { 29 | description = "Size of the VM , exampel Standard_DS4_v2" 30 | default = "Standard_DS4_v2" 31 | type = string 32 | } 33 | 34 | variable storage_image_reference { 35 | type = object({ 36 | publisher = string 37 | offer = string 38 | sku = string 39 | version = string 40 | }) 41 | } 42 | 43 | variable plan { 44 | type = object({ 45 | name = string 46 | publisher = string 47 | product = string 48 | }) 49 | } 50 | 51 | variable zone { 52 | type = string 53 | } 54 | 55 | variable public_ip_address_id { 56 | type = string 57 | } 58 | 59 | variable "admin_username" { 60 | description = "Admin username" 61 | type = string 62 | } 63 | 64 | variable "admin_password" { 65 | description = "Admin password" 66 | sensitive = true 67 | } 68 | 69 | variable "disable_password_authentication" { 70 | description = "disable_password_authentication" 71 | type = bool 72 | } 73 | 74 | variable "ssh_key_data" { 75 | description = "ssh_key_data" 76 | type = string 77 | } 78 | 79 | 80 | -------------------------------------------------------------------------------- /modules/vnet-peering/main.tf: -------------------------------------------------------------------------------- 1 | # enable global peering between the two virtual network 2 | resource "azurerm_virtual_network_peering" "peering12" { 3 | name = "peering-from-${var.peering[0].vnet_name}-to-${var.peering[1].vnet_name}" 4 | resource_group_name = var.resource_group_name 5 | virtual_network_name = var.peering[0].vnet_name 6 | remote_virtual_network_id = var.peering[1].vnet_id 7 | allow_virtual_network_access = var.peering[0].allow_virtual_network_access 8 | allow_forwarded_traffic = var.peering[0].allow_forwarded_traffic 9 | allow_gateway_transit = var.peering[0].allow_gateway_transit 10 | use_remote_gateways = var.peering[0].use_remote_gateways 11 | } 12 | 13 | resource "azurerm_virtual_network_peering" "peering21" { 14 | name = "peering-from-${var.peering[1].vnet_name}-to-${var.peering[0].vnet_name}" 15 | resource_group_name = var.resource_group_name 16 | virtual_network_name = var.peering[1].vnet_name 17 | remote_virtual_network_id = var.peering[0].vnet_id 18 | allow_virtual_network_access = var.peering[1].allow_virtual_network_access 19 | allow_forwarded_traffic = var.peering[1].allow_forwarded_traffic 20 | allow_gateway_transit = var.peering[1].allow_gateway_transit 21 | use_remote_gateways = var.peering[1].use_remote_gateways 22 | } 23 | -------------------------------------------------------------------------------- /modules/vnet-peering/outputs.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/modules/vnet-peering/outputs.tf -------------------------------------------------------------------------------- /modules/vnet-peering/variables.tf: -------------------------------------------------------------------------------- 1 | variable "resource_group_name" { 2 | description = "Name of the resource group." 3 | type = string 4 | } 5 | 6 | variable "location" { 7 | description = "The location of the VM to create. Defaults to the location of the resource group." 8 | type = string 9 | default = null 10 | } 11 | 12 | variable "peering" { 13 | description = "Peered vnets properties" 14 | type = list(object({ 15 | vnet_id = string 16 | vnet_name = string 17 | allow_virtual_network_access = bool 18 | allow_forwarded_traffic = bool 19 | allow_gateway_transit = bool 20 | use_remote_gateways = bool 21 | })) 22 | } 23 | 24 | 25 | -------------------------------------------------------------------------------- /supplementals/HTTP.drawio: -------------------------------------------------------------------------------- 1 | 5Vddb9sgFP01ftxkmzhJHxM3bV9aTUqnbo/U3Nps2FgYJ05//SDGnzTqUmXZpigPgQOcyz3nhoCDwrS6FThP7jkB5vguqRx07fi+57m++tLIrkbmwawGYkGJmdQBa/oKBnQNWlICxWCi5JxJmg/BiGcZRHKAYSH4djjthbNh1BzHYAHrCDMbfaJEJk0WboffAY0T2SZsRlLcTDZAkWDCtz0IrRwUCs5l3UqrEJgWr9GlXndzYLTdmIBM/s6Crw9p+P3xG5k/MR6nd7T88fD8ybBsMCtNwmazctcoABlZaCFVL+OZApeJTJnqeaopeJkR0BFc1avXArEk7fbotZmrkgGeghQ7NWXbadtIm/RkbTABDEu6GdJjY3Hc0rURvnCqAvuuKUd/YnhMMQbIHVIUvBQRmFV9Ld8hQu6ISGIRg7SIVKOXdgftrTrCNv+ibJtcnci2MdG5bUNH2RYxXBQ0+p+da3kawacfdW42JJqc2bnJpTlnCR589Kicjojm53UusJwLGQUTq++fhEqOHIOCvuLn/QTtGS4lL+obih7GjMaZdluxgVDABoSk6uqwMAMpJUQvXuY6uX26wdIJrhXyQhkLOeOiO54LKfhPGIFvFY0OA9WxZdPY4Y/O0sbXd8rKdw9X0MCyY/2ZWv6sQagUL9Sf8e/ur/szs/y5XT3q4FLmDlpocv+m+rz/HPJsm1AJ6xxHGt6qh8LQxxPIhrzRHw2yZfPQG7qNrxIn0+3K0s27X+oXCFXi/DM6tTeiP6CT6navm/o4796IaPUL -------------------------------------------------------------------------------- /supplementals/img/AFWEnableAccelnet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/img/AFWEnableAccelnet.png -------------------------------------------------------------------------------- /supplementals/img/Topology0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/img/Topology0.png -------------------------------------------------------------------------------- /supplementals/img/Topology1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/img/Topology1.png -------------------------------------------------------------------------------- /supplementals/img/firewall policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/img/firewall policy.png -------------------------------------------------------------------------------- /supplementals/img/spoke-to-hub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/img/spoke-to-hub.png -------------------------------------------------------------------------------- /supplementals/log.txt: -------------------------------------------------------------------------------- 1 | 1/19/2022 2 | 3 | 12:30 - 12:45 : vegeta http ; IDPS Alert and Deny ; 50 VMSS ; Reply 1M -> Rate 9Gbps 4 | 5 | 1:20 - 1:25 ; iperf3 ; IDPS Alert and Deny ; 60 VMSS ; -> Rate 9Gbps 6 | 7 | 3:10 - 3:20 iperf3 ; IDPS Alert and Deny ; 60 VMSS -> rate 9Gbps 8 | 9 | 3:40 - 4:00 vegeta http ; IDPS off ; 50 per second ; 60 VMSS -> rate 25Gbps; 1Mb file ; success = 1 10 | 11 | 4:00 - 4:20 vegeta http; IDPS off ; 200 per second ; 60 VMSS -> rate 25Gbps; 1Mb file ; success = 0.25 12 | 13 | 5:20 - 5:30 : vegeta http ; IDPS Alert and Deny ; 60 VMSS ; rate 50 Reply 1M -> Rate 9Gbps 14 | 15 | 5:50 - 6:00 : vegeta http ; IDPS Alert and Deny ; 60 VMSS ; rate 50 Reply 1M -> Rate 25Gbps 16 | 17 | 1/20/2022 18 | 19 | 1:30 - 1:35 vegeta http; IDPS off ;Rate 50 ; 80 VMSS ; Reply 1Mb -> Rate 35Gbps 20 | 21 | 1:50 - 2:00 vegeta http; IDPS off ; Rate 50 ; 80 VMSS ; Reply 10Mb -> Rate 35Gbps 22 | 23 | 2:10 - 2:20 vegeta http; IDPS off ; Rate 200 80 VMSS ; Reply 1Mb -> Rate 8Gbps 24 | 25 | 2:30- 2:40 vegeta http; IDPS A&D ; Rate 50 80 VMSS ; Reply 1Mb -> Rate 9 Gbps 26 | 27 | -insecure 28 | 3:15-3:20 vegeta https; IDPS OFF ; Rate 50 80 VMSS ; Reply 1Mb ; 5 minutes -> Rate 35 Gbps 29 | 30 | 3:40 - 3:45 vegeta https; IDPS A&D ; Rate 50 80 VMSS ; Reply 1Mb ; 5 minutes -> Rate 9 Gbps 31 | 32 | 1kb : IDPS off : ----------------------------------- 33 | 1 VM 34 | 35 | echo "GET http://10.1.0.6" | vegeta attack -duration=10s -rate=10000 | vegeta report 36 | Requests [total, rate, throughput] 100000, 10000.09, 9866.44 37 | Duration [total, attack, wait] 10.135s, 10s, 135.451ms 38 | Latencies [min, mean, 50, 90, 95, 99, max] 777.9µs, 1.774ms, 1.284ms, 2.065ms, 2.469ms, 8.014ms, 404.768ms 39 | Bytes In [total, mean] 138600000, 1386.00 40 | Bytes Out [total, mean] 0, 0.00 41 | Success [ratio] 100.00% 42 | Status Codes [code:count] 200:100000 43 | Error Set: 44 | 45 | echo "GET http://10.1.0.6" | vegeta attack -duration=10s -rate=20000 | vegeta report 46 | Requests [total, rate, throughput] 200000, 19912.13, 8011.35 47 | Duration [total, attack, wait] 10.341s, 10.044s, 297.205ms 48 | Latencies [min, mean, 50, 90, 95, 99, max] 16.3µs, 154.481ms, 166.708ms, 364.812ms, 398.321ms, 522.508ms, 997.885ms 49 | Bytes In [total, mean] 114827328, 574.14 50 | Bytes Out [total, mean] 0, 0.00 51 | Success [ratio] 41.42% 52 | Status Codes [code:count] 0:117152 200:82848 53 | Error Set: 54 | Get "http://10.1.0.6": dial tcp 0.0.0.0:0->10.1.0.6:80: socket: too many open files 55 | 56 | echo "GET http://10.1.0.6" | vegeta attack -duration=60s -rate=10000 | vegeta report 57 | Requests [total, rate, throughput] 600000, 9999.97, 9973.09 58 | Duration [total, attack, wait] 1m0s, 1m0s, 156.236ms 59 | Latencies [min, mean, 50, 90, 95, 99, max] 29.1µs, 2.754ms, 1.511ms, 3.935ms, 5.088ms, 15.48ms, 401.111ms 60 | Bytes In [total, mean] 831523770, 1385.87 61 | Bytes Out [total, mean] 0, 0.00 62 | Success [ratio] 99.99% 63 | Status Codes [code:count] 0:55 200:599945 64 | Error Set: 65 | Get "http://10.1.0.6": dial tcp 0.0.0.0:0->10.1.0.6:80: socket: too many open files 66 | 67 | 68 | 4:50-5:00 vegeta http; IDPS off ; Rate 10000 80 VMSS ; Reply 1kb -> Rate 10-11Gbps: 69 | 70 | 5:05 - 5:20 vegeta http; IDPS A&D ; Rate 10000 80 VMSS ; Reply 1kb -> Rate 4Gbps :::: 71 | 72 | 73 | Standard_D2_v4 https://docs.microsoft.com/en-us/azure/virtual-machines/dv4-dsv4-series 5000Mbps 74 | 75 | D2_v4 rate100+100 / 1M IDPS Off 64 streams 10 min rate 50 - 45 Gbps 76 | 77 | Load balancer 400 rate D4_v4 1M reply IDPS OFF 40 / 160 - not impressed 78 | 79 | Enable accelerated networking D2_v4 rate100+100 / 1M IDPS Off 64 streams 10 min rate 50 - 45 Gbps :-( 80 | 81 | Enable accelerated networking D2_v4 120/70(lb) / 1M IDPS Off 64 streams 10 min rate 50 - 40 Gbps 82 | 83 | 4:50-5:07 1/23 enable accelerated networking D2_v4 120/120 / 1M IDPS Off 64 streams 10 min rate 50 - 55 Gbps 84 | 85 | 5:20 - 5:28 1/23 enable accelerated networking D2_v4 120/120 / 1M IDPS A&D 64 streams 10 min rate 50 - 10-9 Gbps 86 | 87 | 5:41 - 5:59 1/23 enable accelerated networking D2_v4 120/120 / 1M IDPS Alert only 64 streams 10 min rate 50 - 45 Gbps 88 | 89 | -------------------------------------------------------------------------------- /supplementals/stats/IDPS-network-delay-stats.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/IDPS-network-delay-stats.xlsx -------------------------------------------------------------------------------- /supplementals/stats/IDPS-network-stats.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/IDPS-network-stats.xlsx -------------------------------------------------------------------------------- /supplementals/stats/http-latency-http-alert-vs-deny.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/http-latency-http-alert-vs-deny.png -------------------------------------------------------------------------------- /supplementals/stats/latency-IDPS-Alert-D4-v4-iperf3-64-30-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/latency-IDPS-Alert-D4-v4-iperf3-64-30-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/latency-IDPS-Alert-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/latency-IDPS-Alert-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/latency-IDPS-Alert-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/latency-IDPS-Alert-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/latency-IDPS-Alert-and-Deny-D4-v4-iperf3-64-30-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/latency-IDPS-Alert-and-Deny-D4-v4-iperf3-64-30-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/latency-IDPS-Alert-and-Deny-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/latency-IDPS-Alert-and-Deny-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/latency-IDPS-Alert-and-Deny-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/latency-IDPS-Alert-and-Deny-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/latency-IDPS-Off-D4-v4-iperf3-64-30-instances-10-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/latency-IDPS-Off-D4-v4-iperf3-64-30-instances-10-min.png -------------------------------------------------------------------------------- /supplementals/stats/latency-IDPS-Off-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/latency-IDPS-Off-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/latency-IDPS-Off-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/latency-IDPS-Off-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/metrics-1740sec-30instances.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/metrics-1740sec-30instances.png -------------------------------------------------------------------------------- /supplementals/stats/metrics-280sec-30instances.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/metrics-280sec-30instances.png -------------------------------------------------------------------------------- /supplementals/stats/metrics-30sec-30instances.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/metrics-30sec-30instances.png -------------------------------------------------------------------------------- /supplementals/stats/metrics-540sec-120instances-idps-off.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/metrics-540sec-120instances-idps-off.png -------------------------------------------------------------------------------- /supplementals/stats/metrics-IDPS-Alert-D4-v4-iperf3-64-30-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/metrics-IDPS-Alert-D4-v4-iperf3-64-30-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/metrics-IDPS-Alert-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/metrics-IDPS-Alert-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/metrics-IDPS-Alert-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/metrics-IDPS-Alert-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/metrics-IDPS-Alert-and-Deny-D4-v4-iperf3-64-30-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/metrics-IDPS-Alert-and-Deny-D4-v4-iperf3-64-30-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/metrics-IDPS-Alert-and-Deny-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/metrics-IDPS-Alert-and-Deny-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/metrics-IDPS-Alert-and-Deny-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/metrics-IDPS-Alert-and-Deny-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/metrics-IDPS-Off-D4-v4-iperf3-64-30-instances-10-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/metrics-IDPS-Off-D4-v4-iperf3-64-30-instances-10-min.png -------------------------------------------------------------------------------- /supplementals/stats/metrics-IDPS-Off-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/metrics-IDPS-Off-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/metrics-IDPS-Off-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/metrics-IDPS-Off-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/metrics-IDPS-vs-noIDPS-Alert-D4-v4-iperf3-64-30-instances-5-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/metrics-IDPS-vs-noIDPS-Alert-D4-v4-iperf3-64-30-instances-5-min.png -------------------------------------------------------------------------------- /supplementals/stats/stats-1740sec-30instances.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/stats-1740sec-30instances.png -------------------------------------------------------------------------------- /supplementals/stats/stats-1740sec-30instances.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/stats-1740sec-30instances.xlsx -------------------------------------------------------------------------------- /supplementals/stats/stats-280sec-30instances.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/stats-280sec-30instances.png -------------------------------------------------------------------------------- /supplementals/stats/stats-280sec-30instances.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/stats-280sec-30instances.xlsx -------------------------------------------------------------------------------- /supplementals/stats/stats-30sec-30instances.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/stats-30sec-30instances.png -------------------------------------------------------------------------------- /supplementals/stats/stats-30sec-30instances.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/stats-30sec-30instances.xlsx -------------------------------------------------------------------------------- /supplementals/stats/stats-IDPS-Alert-and-Deny-D4-v4-iperf3-64-30-instances-5-min.csv: -------------------------------------------------------------------------------- 1 | PartitionKey,RowKey,test_type,test_type@type,protocol,protocol@type,latency_total,latency_total@type,latency_mean,latency_mean@type,latency_max,latency_max@type,latency_min,latency_min@type,throughput,throughput@type,success,success@type,server,server@type 2 | 10.2.0.10,01 28 2022 12 25 00,vegeta,String,https,String,345187560.1,Double,28765.63001,Double,30035.83136,Double,67.68255,Double,4.310923712,String,0.095666667,String,10.1.0.25,String 3 | 10.2.0.10,01 28 2022 12 30 00,vegeta,String,https,String,344361695.3,Double,28696.80795,Double,30038.58229,Double,59.846101,Double,4.504198059,String,0.0995,String,10.1.0.25,String 4 | 10.2.0.10,01 28 2022 12 35 00,vegeta,String,https,String,345157486.9,Double,28763.12391,Double,30039.7973,Double,62.42097,Double,4.381408087,String,0.097333333,String,10.1.0.25,String 5 | 10.2.0.11,01 28 2022 12 25 00,vegeta,String,https,String,344127435.3,Double,28677.28627,Double,30037.67628,Double,63.381254,Double,4.495565945,String,0.099166667,String,10.1.0.34,String 6 | 10.2.0.11,01 28 2022 12 30 00,vegeta,String,https,String,344994775.9,Double,28749.56466,Double,30034.43077,Double,87.739511,Double,4.274286763,String,0.094416667,String,10.1.0.34,String 7 | 10.2.0.11,01 28 2022 12 35 00,vegeta,String,https,String,345328788.1,Double,28777.399,Double,30036.22284,Double,68.792298,Double,4.284080607,String,0.094583333,String,10.1.0.34,String 8 | 10.2.0.12,01 28 2022 12 25 00,vegeta,String,https,String,344949685.3,Double,28745.80711,Double,30042.47911,Double,74.477628,Double,4.362864535,String,0.09625,String,10.1.0.33,String 9 | 10.2.0.12,01 28 2022 12 30 00,vegeta,String,https,String,344604657,Double,28717.05475,Double,30037.49374,Double,70.396077,Double,4.378884477,String,0.09725,String,10.1.0.33,String 10 | 10.2.0.12,01 28 2022 12 35 00,vegeta,String,https,String,344192601.8,Double,28682.71682,Double,30035.19553,Double,64.362602,Double,4.5243588,String,0.099833333,String,10.1.0.33,String 11 | 10.2.0.13,01 28 2022 12 25 00,vegeta,String,https,String,345940431.7,Double,28828.36931,Double,30031.8843,Double,79.508099,Double,4.099947205,String,0.091333333,String,10.1.0.5,String 12 | 10.2.0.13,01 28 2022 12 30 00,vegeta,String,https,String,345899610,Double,28824.9675,Double,30037.01774,Double,74.51114,Double,4.153928744,String,0.09175,String,10.1.0.5,String 13 | 10.2.0.13,01 28 2022 12 35 00,vegeta,String,https,String,344864144.7,Double,28738.67873,Double,30044.53539,Double,60.139227,Double,4.320183522,String,0.095333333,String,10.1.0.5,String 14 | 10.2.0.14,01 28 2022 12 25 00,vegeta,String,https,String,344941555.5,Double,28745.12963,Double,30038.7242,Double,74.764506,Double,4.308772366,String,0.09525,String,10.1.0.9,String 15 | 10.2.0.14,01 28 2022 12 30 00,vegeta,String,https,String,344858553.5,Double,28738.21279,Double,30029.37629,Double,61.410131,Double,4.375855133,String,0.096666667,String,10.1.0.9,String 16 | 10.2.0.14,01 28 2022 12 35 00,vegeta,String,https,String,345378604.5,Double,28781.55038,Double,30026.21131,Double,65.75791,Double,4.25956931,String,0.094,String,10.1.0.9,String 17 | 10.2.0.15,01 28 2022 12 25 00,vegeta,String,https,String,344831506.3,Double,28735.95886,Double,30037.89022,Double,59.806914,Double,4.367383416,String,0.096333333,String,10.1.0.38,String 18 | 10.2.0.15,01 28 2022 12 30 00,vegeta,String,https,String,345121836.3,Double,28760.15302,Double,30038.88998,Double,85.167715,Double,4.326951295,String,0.095583333,String,10.1.0.38,String 19 | 10.2.0.15,01 28 2022 12 35 00,vegeta,String,https,String,345005260.3,Double,28750.43835,Double,30040.59584,Double,66.017413,Double,4.244834062,String,0.093666667,String,10.1.0.38,String 20 | 10.2.0.16,01 28 2022 12 25 00,vegeta,String,https,String,344674185.3,Double,28722.84877,Double,30030.41281,Double,57.11325,Double,4.412963146,String,0.097916667,String,10.1.0.28,String 21 | 10.2.0.16,01 28 2022 12 30 00,vegeta,String,https,String,344774985.5,Double,28731.2488,Double,30034.68554,Double,76.702892,Double,4.271197496,String,0.095416667,String,10.1.0.28,String 22 | 10.2.0.16,01 28 2022 12 35 00,vegeta,String,https,String,345796471.7,Double,28816.37264,Double,30043.94986,Double,61.739918,Double,4.116626524,String,0.091416667,String,10.1.0.28,String 23 | 10.2.0.17,01 28 2022 12 25 00,vegeta,String,https,String,345124253,Double,28760.35441,Double,30035.30314,Double,74.004471,Double,4.324623505,String,0.095416667,String,10.1.0.23,String 24 | 10.2.0.17,01 28 2022 12 30 00,vegeta,String,https,String,344365202.5,Double,28697.10021,Double,30035.2689,Double,66.718022,Double,4.376402359,String,0.096666667,String,10.1.0.23,String 25 | 10.2.0.17,01 28 2022 12 35 00,vegeta,String,https,String,345695172.8,Double,28807.93106,Double,30034.37902,Double,58.397139,Double,4.244732279,String,0.093666667,String,10.1.0.23,String 26 | 10.2.0.18,01 28 2022 12 25 00,vegeta,String,https,String,345680695.5,Double,28806.72462,Double,30039.01713,Double,63.310021,Double,4.209765745,String,0.092916667,String,10.1.0.18,String 27 | 10.2.0.18,01 28 2022 12 30 00,vegeta,String,https,String,344973764.4,Double,28747.8137,Double,30049.15383,Double,65.303642,Double,4.265497407,String,0.094166667,String,10.1.0.18,String 28 | 10.2.0.18,01 28 2022 12 35 00,vegeta,String,https,String,345469699.9,Double,28789.14166,Double,30044.52379,Double,63.229356,Double,4.17252908,String,0.093833333,String,10.1.0.18,String 29 | 10.2.0.19,01 28 2022 12 25 00,vegeta,String,https,String,345338657.5,Double,28778.22146,Double,30030.31746,Double,71.925359,Double,4.222919526,String,0.093583333,String,10.1.0.16,String 30 | 10.2.0.19,01 28 2022 12 30 00,vegeta,String,https,String,344592467.8,Double,28716.03898,Double,30036.40567,Double,70.585748,Double,4.53379688,String,0.1005,String,10.1.0.16,String 31 | 10.2.0.19,01 28 2022 12 35 00,vegeta,String,https,String,344538098.1,Double,28711.50817,Double,30037.94198,Double,87.019002,Double,4.448166981,String,0.098416667,String,10.1.0.16,String 32 | 10.2.0.20,01 28 2022 12 25 00,vegeta,String,https,String,345247601.8,Double,28770.63348,Double,30033.3186,Double,70.879604,Double,4.386900688,String,0.096916667,String,10.1.0.27,String 33 | 10.2.0.20,01 28 2022 12 30 00,vegeta,String,https,String,344590659.1,Double,28715.88826,Double,30036.62096,Double,66.850071,Double,4.312591029,String,0.09625,String,10.1.0.27,String 34 | 10.2.0.20,01 28 2022 12 35 00,vegeta,String,https,String,345814860.7,Double,28817.90506,Double,30034.24646,Double,60.298871,Double,4.078964545,String,0.091083333,String,10.1.0.27,String 35 | 10.2.0.21,01 28 2022 12 25 00,vegeta,String,https,String,344874544.8,Double,28739.5454,Double,30036.63069,Double,67.885766,Double,4.28039162,String,0.095916667,String,10.1.0.20,String 36 | 10.2.0.21,01 28 2022 12 30 00,vegeta,String,https,String,344745696.9,Double,28728.80807,Double,30038.31227,Double,82.781329,Double,4.403695076,String,0.09725,String,10.1.0.20,String 37 | 10.2.0.21,01 28 2022 12 35 00,vegeta,String,https,String,345643832.5,Double,28803.65271,Double,30042.02819,Double,67.290505,Double,4.187052298,String,0.0925,String,10.1.0.20,String 38 | 10.2.0.22,01 28 2022 12 25 00,vegeta,String,https,String,344499325.5,Double,28708.27713,Double,30037.64581,Double,65.206662,Double,4.435431403,String,0.097833333,String,10.1.0.10,String 39 | 10.2.0.22,01 28 2022 12 30 00,vegeta,String,https,String,345186212.5,Double,28765.5177,Double,30036.36482,Double,58.449788,Double,4.299041475,String,0.095166667,String,10.1.0.10,String 40 | 10.2.0.22,01 28 2022 12 35 00,vegeta,String,https,String,345355933.5,Double,28779.66112,Double,30041.94507,Double,73.586408,Double,4.315057648,String,0.095333333,String,10.1.0.10,String 41 | 10.2.0.23,01 28 2022 12 25 00,vegeta,String,https,String,345790127.6,Double,28815.84397,Double,30035.60023,Double,78.509445,Double,4.27599451,String,0.094333333,String,10.1.0.29,String 42 | 10.2.0.23,01 28 2022 12 30 00,vegeta,String,https,String,345083081.6,Double,28756.92347,Double,30038.68352,Double,79.283435,Double,4.281739003,String,0.095083333,String,10.1.0.29,String 43 | 10.2.0.23,01 28 2022 12 35 00,vegeta,String,https,String,345504435.8,Double,28792.03632,Double,30036.21061,Double,58.125631,Double,4.26283776,String,0.094,String,10.1.0.29,String 44 | 10.2.0.24,01 28 2022 12 25 00,vegeta,String,https,String,345013530.2,Double,28751.12751,Double,30037.4632,Double,61.673357,Double,4.435197076,String,0.09875,String,10.1.0.22,String 45 | 10.2.0.24,01 28 2022 12 30 00,vegeta,String,https,String,344659449.2,Double,28721.62076,Double,30037.85355,Double,67.158291,Double,4.478284528,String,0.098916667,String,10.1.0.22,String 46 | 10.2.0.24,01 28 2022 12 35 00,vegeta,String,https,String,345177532.5,Double,28764.79438,Double,30032.59385,Double,77.054918,Double,4.215409706,String,0.094083333,String,10.1.0.22,String 47 | 10.2.0.25,01 28 2022 12 25 00,vegeta,String,https,String,345448734.4,Double,28787.39453,Double,30029.65679,Double,69.466994,Double,4.322764313,String,0.095333333,String,10.1.0.15,String 48 | 10.2.0.25,01 28 2022 12 30 00,vegeta,String,https,String,344983825.9,Double,28748.65216,Double,30037.85068,Double,67.636597,Double,4.414170008,String,0.0975,String,10.1.0.15,String 49 | 10.2.0.25,01 28 2022 12 35 00,vegeta,String,https,String,345890999.1,Double,28824.24992,Double,30039.67024,Double,79.243552,Double,4.250920435,String,0.093916667,String,10.1.0.15,String 50 | 10.2.0.26,01 28 2022 12 25 00,vegeta,String,https,String,345077184.7,Double,28756.43206,Double,30033.69265,Double,68.221485,Double,4.318492742,String,0.09525,String,10.1.0.6,String 51 | 10.2.0.26,01 28 2022 12 30 00,vegeta,String,https,String,344753022.7,Double,28729.41856,Double,30043.19656,Double,67.167003,Double,4.274404038,String,0.094416667,String,10.1.0.6,String 52 | 10.2.0.26,01 28 2022 12 35 00,vegeta,String,https,String,345713019.5,Double,28809.41829,Double,30041.24496,Double,67.748507,Double,4.19564302,String,0.092583333,String,10.1.0.6,String 53 | 10.2.0.27,01 28 2022 12 25 00,vegeta,String,https,String,345461895.2,Double,28788.49127,Double,30040.91269,Double,66.603211,Double,4.177914619,String,0.092166667,String,10.1.0.37,String 54 | 10.2.0.27,01 28 2022 12 30 00,vegeta,String,https,String,344245273.4,Double,28687.10611,Double,30039.20289,Double,69.709056,Double,4.247301195,String,0.094,String,10.1.0.37,String 55 | 10.2.0.27,01 28 2022 12 35 00,vegeta,String,https,String,345633146.3,Double,28802.76219,Double,30028.73948,Double,73.650871,Double,4.109193125,String,0.09125,String,10.1.0.37,String 56 | 10.2.0.28,01 28 2022 12 25 00,vegeta,String,https,String,345375249.3,Double,28781.27078,Double,30036.87015,Double,64.430081,Double,4.202681669,String,0.093666667,String,10.1.0.14,String 57 | 10.2.0.28,01 28 2022 12 30 00,vegeta,String,https,String,345099989.5,Double,28758.33246,Double,30038.10495,Double,71.219652,Double,4.353093635,String,0.096166667,String,10.1.0.14,String 58 | 10.2.0.28,01 28 2022 12 35 00,vegeta,String,https,String,346014031.7,Double,28834.50264,Double,30032.64212,Double,73.645538,Double,4.052414154,String,0.089416667,String,10.1.0.14,String 59 | 10.2.0.30,01 28 2022 12 25 00,vegeta,String,https,String,345418911,Double,28784.90925,Double,30038.90539,Double,75.345062,Double,4.144400586,String,0.093083333,String,10.1.0.13,String 60 | 10.2.0.30,01 28 2022 12 30 00,vegeta,String,https,String,345360574.9,Double,28780.04791,Double,30037.06226,Double,72.450954,Double,4.215049662,String,0.094583333,String,10.1.0.13,String 61 | 10.2.0.30,01 28 2022 12 35 00,vegeta,String,https,String,346047843.5,Double,28837.3203,Double,30040.17394,Double,64.628602,Double,4.135083045,String,0.09125,String,10.1.0.13,String 62 | 10.2.0.31,01 28 2022 12 25 00,vegeta,String,https,String,345018743.2,Double,28751.56193,Double,30042.85665,Double,71.732157,Double,4.427549547,String,0.097666667,String,10.1.0.8,String 63 | 10.2.0.31,01 28 2022 12 30 00,vegeta,String,https,String,344867347,Double,28738.94558,Double,30039.92038,Double,61.487157,Double,4.402803631,String,0.09725,String,10.1.0.8,String 64 | 10.2.0.31,01 28 2022 12 35 00,vegeta,String,https,String,345602870.5,Double,28800.23921,Double,30042.94365,Double,68.262978,Double,4.153952212,String,0.091666667,String,10.1.0.8,String 65 | 10.2.0.33,01 28 2022 12 25 00,vegeta,String,https,String,344543427.8,Double,28711.95232,Double,30039.18498,Double,76.732891,Double,4.504581045,String,0.099333333,String,10.1.0.12,String 66 | 10.2.0.33,01 28 2022 12 30 00,vegeta,String,https,String,344654381.6,Double,28721.19847,Double,30033.1835,Double,63.587158,Double,4.493244257,String,0.09925,String,10.1.0.12,String 67 | 10.2.0.33,01 28 2022 12 35 00,vegeta,String,https,String,344922543.4,Double,28743.54529,Double,30039.79874,Double,79.151991,Double,4.37532785,String,0.096666667,String,10.1.0.12,String 68 | 10.2.0.38,01 28 2022 12 25 00,vegeta,String,https,String,345127380.1,Double,28760.61501,Double,30035.59171,Double,66.822063,Double,4.287206582,String,0.094916667,String,10.1.0.31,String 69 | 10.2.0.38,01 28 2022 12 30 00,vegeta,String,https,String,343551382.7,Double,28629.28189,Double,30040.47801,Double,63.150367,Double,4.653570509,String,0.104666667,String,10.1.0.31,String 70 | 10.2.0.38,01 28 2022 12 35 00,vegeta,String,https,String,344062130.6,Double,28671.84421,Double,30045.13355,Double,69.793284,Double,4.537404572,String,0.10025,String,10.1.0.31,String 71 | 10.2.0.39,01 28 2022 12 25 00,vegeta,String,https,String,345008220.8,Double,28750.68507,Double,30041.36176,Double,94.428781,Double,4.382752012,String,0.096666667,String,10.1.0.19,String 72 | 10.2.0.39,01 28 2022 12 30 00,vegeta,String,https,String,344275441.5,Double,28689.62013,Double,30039.62439,Double,67.770988,Double,4.419643256,String,0.097583333,String,10.1.0.19,String 73 | 10.2.0.39,01 28 2022 12 35 00,vegeta,String,https,String,345316035.1,Double,28776.33626,Double,30033.12787,Double,62.518201,Double,4.294832923,String,0.09475,String,10.1.0.19,String 74 | 10.2.0.40,01 28 2022 12 25 00,vegeta,String,https,String,344966763.9,Double,28747.23033,Double,30038.9586,Double,54.666558,Double,4.273373107,String,0.094583333,String,10.1.0.39,String 75 | 10.2.0.40,01 28 2022 12 30 00,vegeta,String,https,String,344195364.7,Double,28682.94706,Double,30041.29545,Double,63.195231,Double,4.585771533,String,0.101666667,String,10.1.0.39,String 76 | 10.2.0.40,01 28 2022 12 35 00,vegeta,String,https,String,343799149,Double,28649.92908,Double,30039.31571,Double,69.672038,Double,4.598925331,String,0.10175,String,10.1.0.39,String 77 | 10.2.0.5,01 28 2022 12 25 00,vegeta,String,https,String,345451703.1,Double,28787.64192,Double,30041.7258,Double,69.884462,Double,4.351386321,String,0.096,String,10.1.0.32,String 78 | 10.2.0.5,01 28 2022 12 30 00,vegeta,String,https,String,345291157.3,Double,28774.26311,Double,30041.51267,Double,63.634124,Double,4.2889776,String,0.09475,String,10.1.0.32,String 79 | 10.2.0.5,01 28 2022 12 35 00,vegeta,String,https,String,345484163.9,Double,28790.34699,Double,30029.18939,Double,78.690273,Double,4.14453165,String,0.092333333,String,10.1.0.32,String 80 | 10.2.0.6,01 28 2022 12 25 00,vegeta,String,https,String,344742102.2,Double,28728.50852,Double,30039.95902,Double,69.581427,Double,4.332340881,String,0.095583333,String,10.1.0.24,String 81 | 10.2.0.6,01 28 2022 12 30 00,vegeta,String,https,String,345042183,Double,28753.51525,Double,30042.0173,Double,69.518387,Double,4.293625443,String,0.094833333,String,10.1.0.24,String 82 | 10.2.0.6,01 28 2022 12 35 00,vegeta,String,https,String,345683097.9,Double,28806.92482,Double,30037.12734,Double,64.345915,Double,4.085495729,String,0.091583333,String,10.1.0.24,String 83 | 10.2.0.7,01 28 2022 12 25 00,vegeta,String,https,String,344728622.3,Double,28727.38519,Double,30043.66405,Double,63.775804,Double,4.465948343,String,0.0985,String,10.1.0.17,String 84 | 10.2.0.7,01 28 2022 12 30 00,vegeta,String,https,String,344073866.3,Double,28672.82219,Double,30040.04187,Double,65.687856,Double,4.45338749,String,0.098666667,String,10.1.0.17,String 85 | 10.2.0.7,01 28 2022 12 35 00,vegeta,String,https,String,345364926.3,Double,28780.41053,Double,30044.16836,Double,68.390578,Double,4.221130539,String,0.09325,String,10.1.0.17,String 86 | 10.2.0.8,01 28 2022 12 25 00,vegeta,String,https,String,345105674.1,Double,28758.80617,Double,30040.16648,Double,78.232899,Double,4.428426769,String,0.098333333,String,10.1.0.7,String 87 | 10.2.0.8,01 28 2022 12 30 00,vegeta,String,https,String,344327707.2,Double,28693.9756,Double,30041.45412,Double,75.98643,Double,4.379348861,String,0.096666667,String,10.1.0.7,String 88 | 10.2.0.8,01 28 2022 12 35 00,vegeta,String,https,String,345446280,Double,28787.19,Double,30037.59552,Double,68.702471,Double,4.268831907,String,0.094583333,String,10.1.0.7,String 89 | 10.2.0.9,01 28 2022 12 25 00,vegeta,String,https,String,344913735.3,Double,28742.81127,Double,30045.46932,Double,69.030693,Double,4.260982243,String,0.094,String,10.1.0.36,String 90 | 10.2.0.9,01 28 2022 12 30 00,vegeta,String,https,String,342985438.6,Double,28582.11989,Double,30033.05015,Double,75.028884,Double,4.83237446,String,0.10675,String,10.1.0.36,String 91 | 10.2.0.9,01 28 2022 12 35 00,vegeta,String,https,String,343540773.6,Double,28628.3978,Double,30039.29116,Double,71.274361,Double,4.758981199,String,0.105,String,10.1.0.36,String 92 | ,,,,,,,,,,,,,,,,,,, 93 | ,,,,,,,,2587547.748,,,,,,,,,,, 94 | -------------------------------------------------------------------------------- /supplementals/stats/stats-IDPS-Off-D4-v4-iperf3-64-30-instances-10-min.csv: -------------------------------------------------------------------------------- 1 | PartitionKey,RowKey,test_type,test_type@type,sent_throughput,sent_throughput@type,received_throughput,received_throughput@type,max_rtt,max_rtt@type,min_rtt,min_rtt@type,mean_rtt,mean_rtt@type,server,server@type,duration,duration@type,protocol,protocol@type,latency_total,latency_total@type,latency_mean,latency_mean@type,latency_max,latency_max@type,latency_min,latency_min@type,throughput,throughput@type,success,success@type 2 | 10.2.0.10,01 27 2022 13 00 00,iperf3,String,"4,732,364,067",String,"4,728,218,744",String,159724,Int32,7160,Int32,113443,Int32,10.1.0.8,String,5,Int64,,,,,,,,,,,,,, 3 | 10.2.0.13,01 27 2022 13 00 00,iperf3,String,"4,749,249,152",String,"4,745,040,114",String,57308,Int32,3467,Int32,15711,Int32,10.1.0.17,String,5,Int64,,,,,,,,,,,,,, 4 | 10.2.0.15,01 27 2022 13 00 00,iperf3,String,"4,736,490,372",String,"4,732,275,409",String,503058,Int32,3068,Int32,20353,Int32,10.1.0.32,String,5,Int64,,,,,,,,,,,,,, 5 | 10.2.0.16,01 27 2022 13 00 00,iperf3,String,"4,746,712,294",String,"4,742,782,360",String,46385,Int32,3797,Int32,13678,Int32,10.1.0.38,String,5,Int64,,,,,,,,,,,,,, 6 | 10.2.0.17,01 27 2022 13 00 00,iperf3,String,"4,749,254,294",String,"4,739,465,881",String,114380,Int32,7070,Int32,60639,Int32,10.1.0.6,String,5,Int64,,,,,,,,,,,,,, 7 | 10.2.0.18,01 27 2022 13 00 00,iperf3,String,"4,747,524,132",String,"4,741,940,966",String,52709,Int32,5992,Int32,19520,Int32,10.1.0.24,String,5,Int64,,,,,,,,,,,,,, 8 | 10.2.0.19,01 27 2022 13 00 00,iperf3,String,"4,726,699,437",String,"4,723,123,861",String,92469,Int32,27695,Int32,61313,Int32,10.1.0.18,String,5,Int64,,,,,,,,,,,,,, 9 | 10.2.0.20,01 27 2022 13 00 00,iperf3,String,"4,749,878,468",String,"4,745,363,299",String,69756,Int32,3647,Int32,27621,Int32,10.1.0.37,String,5,Int64,,,,,,,,,,,,,, 10 | 10.2.0.21,01 27 2022 13 00 00,iperf3,String,"4,742,848,578",String,"4,738,958,375",String,87510,Int32,7376,Int32,32126,Int32,10.1.0.16,String,5,Int64,,,,,,,,,,,,,, 11 | 10.2.0.22,01 27 2022 13 00 00,iperf3,String,"4,724,484,263",String,"4,720,081,835",String,33057,Int32,6102,Int32,19619,Int32,10.1.0.40,String,5,Int64,,,,,,,,,,,,,, 12 | 10.2.0.23,01 27 2022 13 00 00,iperf3,String,"4,731,677,200",String,"4,727,175,554",String,158715,Int32,10757,Int32,114854,Int32,10.1.0.5,String,5,Int64,,,,,,,,,,,,,, 13 | 10.2.0.26,01 27 2022 13 00 00,iperf3,String,"4,750,552,472",String,"4,746,148,798",String,49100,Int32,5867,Int32,21539,Int32,10.1.0.35,String,5,Int64,,,,,,,,,,,,,, 14 | 10.2.0.27,01 27 2022 13 00 00,iperf3,String,"4,746,897,180",String,"4,742,302,336",String,70128,Int32,9987,Int32,36987,Int32,10.1.0.23,String,5,Int64,,,,,,,,,,,,,, 15 | 10.2.0.28,01 27 2022 13 00 00,iperf3,String,"4,750,323,104",String,"4,745,185,053",String,676405,Int32,10112,Int32,68139,Int32,10.1.0.39,String,5,Int64,,,,,,,,,,,,,, 16 | 10.2.0.29,01 27 2022 13 00 00,iperf3,String,"4,750,513,310",String,"4,739,748,963",String,112569,Int32,6662,Int32,68713,Int32,10.1.0.21,String,5,Int64,,,,,,,,,,,,,, 17 | 10.2.0.31,01 27 2022 13 00 00,iperf3,String,"4,733,106,134",String,"4,729,498,094",String,76186,Int32,8939,Int32,23989,Int32,10.1.0.12,String,5,Int64,,,,,,,,,,,,,, 18 | 10.2.0.33,01 27 2022 13 00 00,iperf3,String,"4,717,024,277",String,"4,712,704,236",String,277441,Int32,6613,Int32,26564,Int32,10.1.0.26,String,5,Int64,,,,,,,,,,,,,, 19 | 10.2.0.34,01 27 2022 13 00 00,iperf3,String,"4,739,099,049",String,"4,734,783,738",String,868047,Int32,3274,Int32,63050,Int32,10.1.0.25,String,5,Int64,,,,,,,,,,,,,, 20 | 10.2.0.35,01 27 2022 13 00 00,iperf3,String,"4,740,978,079",String,"4,736,520,893",String,82092,Int32,6449,Int32,38521,Int32,10.1.0.19,String,5,Int64,,,,,,,,,,,,,, 21 | 10.2.0.36,01 27 2022 13 00 00,iperf3,String,"4,728,050,017",String,"4,724,106,821",String,31737,Int32,3301,Int32,14557,Int32,10.1.0.20,String,5,Int64,,,,,,,,,,,,,, 22 | 10.2.0.37,01 27 2022 13 00 00,iperf3,String,"4,749,292,263",String,"4,744,299,588",String,195119,Int32,8881,Int32,37667,Int32,10.1.0.9,String,5,Int64,,,,,,,,,,,,,, 23 | 10.2.0.38,01 27 2022 13 00 00,iperf3,String,"4,748,364,388",String,"4,744,361,945",String,55092,Int32,2344,Int32,9971,Int32,10.1.0.14,String,5,Int64,,,,,,,,,,,,,, 24 | 10.2.0.40,01 27 2022 13 00 00,iperf3,String,"4,749,252,216",String,"4,744,726,514",String,30729,Int32,13816,Int32,24126,Int32,10.1.0.7,String,5,Int64,,,,,,,,,,,,,, 25 | 10.2.0.41,01 27 2022 13 00 00,iperf3,String,"4,746,825,396",String,"4,742,656,471",String,40497,Int32,8176,Int32,16874,Int32,10.1.0.36,String,5,Int64,,,,,,,,,,,,,, 26 | 10.2.0.6,01 27 2022 13 00 00,iperf3,String,"4,750,263,802",String,"4,745,621,963",String,46563,Int32,6582,Int32,20654,Int32,10.1.0.31,String,5,Int64,,,,,,,,,,,,,, 27 | 10.2.0.7,01 27 2022 13 00 00,iperf3,String,"4,749,814,028",String,"4,745,874,263",String,220116,Int32,4168,Int32,38613,Int32,10.1.0.22,String,5,Int64,,,,,,,,,,,,,, 28 | 10.2.0.8,01 27 2022 13 00 00,iperf3,String,"4,751,058,951",String,"4,747,053,030",String,47215,Int32,6007,Int32,17037,Int32,10.1.0.15,String,5,Int64,,,,,,,,,,,,,, 29 | 10.2.0.9,01 27 2022 13 00 00,iperf3,String,"4,745,540,448",String,"4,741,323,159",String,36608,Int32,1990,Int32,6049,Int32,10.1.0.41,String,5,Int64,,,,,,,,,,,,,, 30 | ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 31 | ,,,,"132,784,137,370",,"132,651,342,263",,,,,,36854.53571,,,,,,,,,,,,,,,,,,, 32 | -------------------------------------------------------------------------------- /supplementals/stats/stats-IDPS-Off-D4-v4-vegeta-http-rate-50-1M-reply-120-instances-5-min.csv: -------------------------------------------------------------------------------- 1 | PartitionKey,RowKey,test_type,test_type@type,protocol,protocol@type,latency_total,latency_total@type,latency_mean,latency_mean@type,latency_max,latency_max@type,latency_min,latency_min@type,throughput,throughput@type,success,success@type,server,server@type 2 | 10.2.0.100,01 24 2022 09 25 00,vegeta,String,http,String,1632512.481,Double,136.042706,Double,5314.661348,Double,50.017089,Double,49.83076146,String,1,String,10.1.0.18,String 3 | 10.2.0.101,01 24 2022 09 25 00,vegeta,String,http,String,1257176.074,Double,104.764672,Double,3641.281952,Double,46.278536,Double,49.97755474,String,1,String,10.1.0.137,String 4 | 10.2.0.102,01 24 2022 09 25 00,vegeta,String,http,String,3635820.136,Double,302.985011,Double,6931.505902,Double,60.1311,Double,49.88476538,String,1,String,10.1.0.118,String 5 | 10.2.0.103,01 24 2022 09 25 00,vegeta,String,http,String,1635018.334,Double,136.251527,Double,16795.66508,Double,48.452296,Double,49.87984805,String,1,String,10.1.0.100,String 6 | 10.2.0.104,01 24 2022 09 25 00,vegeta,String,http,String,1480815.945,Double,123.401328,Double,13291.592,Double,49.460342,Double,49.86273327,String,1,String,10.1.0.127,String 7 | 10.2.0.105,01 24 2022 09 25 00,vegeta,String,http,String,3267315.991,Double,272.276332,Double,4788.189695,Double,59.1532,Double,49.92216052,String,1,String,10.1.0.119,String 8 | 10.2.0.106,01 24 2022 09 25 00,vegeta,String,http,String,1470133.913,Double,122.511159,Double,5069.106372,Double,49.4827,Double,49.8871634,String,1,String,10.1.0.114,String 9 | 10.2.0.107,01 24 2022 09 25 00,vegeta,String,http,String,3414839.838,Double,284.569986,Double,5823.5736,Double,60.3225,Double,49.55443097,String,1,String,10.1.0.105,String 10 | 10.2.0.108,01 24 2022 09 25 00,vegeta,String,http,String,2041278.405,Double,170.106533,Double,12632.18203,Double,48.375511,Double,49.82861098,String,1,String,10.1.0.97,String 11 | 10.2.0.109,01 24 2022 09 25 00,vegeta,String,http,String,4818161.792,Double,401.513482,Double,4721.781856,Double,58.0067,Double,49.91992793,String,1,String,10.1.0.109,String 12 | 10.2.0.11,01 24 2022 09 25 00,vegeta,String,http,String,2005977.328,Double,167.164777,Double,5988.609309,Double,50.508746,Double,49.82302183,String,1,String,10.1.0.89,String 13 | 10.2.0.110,01 24 2022 09 25 00,vegeta,String,http,String,1652423.35,Double,137.701945,Double,5442.065638,Double,49.209913,Double,49.8979477,String,1,String,10.1.0.82,String 14 | 10.2.0.111,01 24 2022 09 25 00,vegeta,String,http,String,3208115.607,Double,267.342967,Double,10882.40975,Double,51.1435,Double,49.83433752,String,1,String,10.1.0.115,String 15 | 10.2.0.112,01 24 2022 09 25 00,vegeta,String,http,String,1514292.182,Double,126.191015,Double,18588.65722,Double,48.13312,Double,49.89676844,String,1,String,10.1.0.33,String 16 | 10.2.0.113,01 24 2022 09 25 00,vegeta,String,http,String,1654676.053,Double,137.889671,Double,8427.427551,Double,48.6179,Double,49.87862123,String,1,String,10.1.0.101,String 17 | 10.2.0.114,01 24 2022 09 25 00,vegeta,String,http,String,4214231.226,Double,351.185935,Double,9070.090601,Double,57.0583,Double,49.93600697,String,1,String,10.1.0.107,String 18 | 10.2.0.115,01 24 2022 09 25 00,vegeta,String,http,String,2271391.962,Double,189.282663,Double,3463.880056,Double,50.6989,Double,49.90824675,String,1,String,10.1.0.29,String 19 | 10.2.0.116,01 24 2022 09 25 00,vegeta,String,http,String,2909704.667,Double,242.475388,Double,2759.718232,Double,56.4658,Double,49.91272758,String,1,String,10.1.0.113,String 20 | 10.2.0.117,01 24 2022 09 25 00,vegeta,String,http,String,1319714.193,Double,109.976182,Double,3990.649769,Double,49.388,Double,49.86484962,String,1,String,10.1.0.94,String 21 | 10.2.0.118,01 24 2022 09 25 00,vegeta,String,http,String,1663899.706,Double,138.658308,Double,7540.482893,Double,49.634,Double,49.88299094,String,1,String,10.1.0.135,String 22 | 10.2.0.118,01 24 2022 09 30 00,vegeta,String,http,String,61526451.46,Double,5127.204288,Double,30000.65593,Double,0.060402,Double,14.61543094,String,0.312916667,String,10.1.0.135,String 23 | 10.2.0.119,01 24 2022 09 25 00,vegeta,String,http,String,1483170.504,Double,123.597541,Double,1160.294873,Double,49.4731,Double,49.90528335,String,1,String,10.1.0.51,String 24 | 10.2.0.12,01 24 2022 09 25 00,vegeta,String,http,String,3533110.505,Double,294.425875,Double,3288.737,Double,54.9219,Double,49.89749887,String,1,String,10.1.0.106,String 25 | 10.2.0.120,01 24 2022 09 25 00,vegeta,String,http,String,1460491.887,Double,121.707657,Double,4166.53839,Double,48.286501,Double,49.96427594,String,1,String,10.1.0.96,String 26 | 10.2.0.121,01 24 2022 09 25 00,vegeta,String,http,String,3302515.768,Double,275.209647,Double,6440.607792,Double,54.2246,Double,49.88202535,String,1,String,10.1.0.111,String 27 | 10.2.0.122,01 24 2022 09 25 00,vegeta,String,http,String,2325413.565,Double,193.784463,Double,4963.418341,Double,54.406003,Double,49.89637946,String,1,String,10.1.0.103,String 28 | 10.2.0.123,01 24 2022 09 25 00,vegeta,String,http,String,1642087.429,Double,136.840619,Double,27964.89406,Double,49.61019,Double,49.98654917,String,1,String,10.1.0.11,String 29 | 10.2.0.124,01 24 2022 09 25 00,vegeta,String,http,String,4453520.514,Double,371.126709,Double,7312.325958,Double,51.9079,Double,49.50748073,String,1,String,10.1.0.121,String 30 | 10.2.0.125,01 24 2022 09 25 00,vegeta,String,http,String,1443484.748,Double,120.290395,Double,4133.498936,Double,49.497032,Double,49.98902034,String,1,String,10.1.0.31,String 31 | 10.2.0.126,01 24 2022 09 25 00,vegeta,String,http,String,3415829.435,Double,284.652452,Double,5516.824574,Double,52.2667,Double,49.87829084,String,1,String,10.1.0.116,String 32 | 10.2.0.127,01 24 2022 09 25 00,vegeta,String,http,String,1719254.448,Double,143.271204,Double,3966.3298,Double,49.850292,Double,49.97886147,String,1,String,10.1.0.40,String 33 | 10.2.0.128,01 24 2022 09 25 00,vegeta,String,http,String,1542517.724,Double,128.543143,Double,2186.325781,Double,48.3293,Double,49.97352275,String,1,String,10.1.0.71,String 34 | 10.2.0.129,01 24 2022 09 25 00,vegeta,String,http,String,10847604.32,Double,903.967026,Double,10710.01744,Double,54.5489,Double,49.78282353,String,1,String,10.1.0.125,String 35 | 10.2.0.13,01 24 2022 09 25 00,vegeta,String,http,String,1437511.267,Double,119.792605,Double,3265.7837,Double,49.6699,Double,49.86947658,String,1,String,10.1.0.56,String 36 | 10.2.0.130,01 24 2022 09 25 00,vegeta,String,http,String,1589508.905,Double,132.459075,Double,3093.914679,Double,49.814054,Double,49.83511012,String,1,String,10.1.0.67,String 37 | 10.2.0.131,01 24 2022 09 25 00,vegeta,String,http,String,1627401.835,Double,135.616819,Double,10526.98666,Double,49.451945,Double,49.84072696,String,1,String,10.1.0.139,String 38 | 10.2.0.132,01 24 2022 09 25 00,vegeta,String,http,String,1781279.156,Double,148.439929,Double,6353.275645,Double,50.1435,Double,49.82103859,String,1,String,10.1.0.65,String 39 | 10.2.0.133,01 24 2022 09 25 00,vegeta,String,http,String,1759125.096,Double,146.593758,Double,7775.161691,Double,49.986008,Double,49.86329454,String,1,String,10.1.0.14,String 40 | 10.2.0.134,01 24 2022 09 25 00,vegeta,String,http,String,3508213.515,Double,292.351126,Double,9679.924728,Double,55.7011,Double,49.89891795,String,1,String,10.1.0.120,String 41 | 10.2.0.135,01 24 2022 09 25 00,vegeta,String,http,String,1681515.114,Double,140.126259,Double,19654.07786,Double,49.308541,Double,48.84481063,String,1,String,10.1.0.99,String 42 | 10.2.0.136,01 24 2022 09 25 00,vegeta,String,http,String,1287286.522,Double,107.273876,Double,4613.513087,Double,49.295703,Double,49.98249225,String,1,String,10.1.0.110,String 43 | 10.2.0.137,01 24 2022 09 25 00,vegeta,String,http,String,1665616.382,Double,138.801365,Double,3040.734164,Double,48.951377,Double,49.89788756,String,1,String,10.1.0.13,String 44 | 10.2.0.138,01 24 2022 09 25 00,vegeta,String,http,String,1694793.13,Double,141.23276,Double,7241.83659,Double,49.425741,Double,49.82006103,String,1,String,10.1.0.12,String 45 | 10.2.0.139,01 24 2022 09 25 00,vegeta,String,http,String,1472128.478,Double,122.677373,Double,2797.041592,Double,49.563873,Double,49.98894611,String,1,String,10.1.0.108,String 46 | 10.2.0.14,01 24 2022 09 25 00,vegeta,String,http,String,3021242.834,Double,251.770236,Double,25324.29103,Double,57.8121,Double,49.86814909,String,1,String,10.1.0.112,String 47 | 10.2.0.15,01 24 2022 09 25 00,vegeta,String,http,String,1809504.56,Double,150.792046,Double,11113.08289,Double,50.064133,Double,49.98165422,String,1,String,10.1.0.50,String 48 | 10.2.0.17,01 24 2022 09 25 00,vegeta,String,http,String,1460546.196,Double,121.712183,Double,14761.6353,Double,48.5273,Double,49.96742673,String,1,String,10.1.0.98,String 49 | 10.2.0.18,01 24 2022 09 25 00,vegeta,String,http,String,1754151.916,Double,146.179326,Double,9220.842104,Double,50.158549,Double,49.86200555,String,1,String,10.1.0.35,String 50 | 10.2.0.19,01 24 2022 09 25 00,vegeta,String,http,String,1501126.179,Double,125.093848,Double,8049.565365,Double,48.890256,Double,49.9013805,String,1,String,10.1.0.38,String 51 | 10.2.0.20,01 24 2022 09 25 00,vegeta,String,http,String,1797691.352,Double,149.807612,Double,6576.429252,Double,50.4822,Double,49.86918173,String,1,String,10.1.0.90,String 52 | 10.2.0.22,01 24 2022 09 25 00,vegeta,String,http,String,1310136.47,Double,109.178039,Double,18053.7406,Double,46.283447,Double,49.97272827,String,1,String,10.1.0.73,String 53 | 10.2.0.23,01 24 2022 09 25 00,vegeta,String,http,String,2598933.507,Double,216.577792,Double,7144.001182,Double,51.8524,Double,49.73735511,String,1,String,10.1.0.28,String 54 | 10.2.0.24,01 24 2022 09 25 00,vegeta,String,http,String,1640734.152,Double,136.727845,Double,6994.787,Double,49.5026,Double,49.9848006,String,1,String,10.1.0.37,String 55 | 10.2.0.25,01 24 2022 09 25 00,vegeta,String,http,String,1560880.733,Double,130.073394,Double,7123.0682,Double,49.7251,Double,49.88266029,String,1,String,10.1.0.36,String 56 | 10.2.0.27,01 24 2022 09 25 00,vegeta,String,http,String,2391013.854,Double,199.251154,Double,7065.8259,Double,55.0557,Double,49.8352385,String,1,String,10.1.0.52,String 57 | 10.2.0.28,01 24 2022 09 25 00,vegeta,String,http,String,3622413.057,Double,301.867754,Double,7216.092,Double,55.0459,Double,49.94944154,String,1,String,10.1.0.126,String 58 | 10.2.0.30,01 24 2022 09 25 00,vegeta,String,http,String,1885088.442,Double,157.090703,Double,3405.4517,Double,50.101402,Double,49.89761113,String,1,String,10.1.0.66,String 59 | 10.2.0.32,01 24 2022 09 25 00,vegeta,String,http,String,2594536.574,Double,216.211381,Double,6471.8616,Double,51.1039,Double,49.91493647,String,1,String,10.1.0.27,String 60 | 10.2.0.33,01 24 2022 09 25 00,vegeta,String,http,String,1837916.063,Double,153.159671,Double,5531.656817,Double,50.0286,Double,49.89779624,String,1,String,10.1.0.85,String 61 | 10.2.0.35,01 24 2022 09 25 00,vegeta,String,http,String,1493301.121,Double,124.44176,Double,4341.8367,Double,49.696933,Double,49.83759019,String,1,String,10.1.0.6,String 62 | 10.2.0.37,01 24 2022 09 25 00,vegeta,String,http,String,1731177.278,Double,144.264773,Double,13258.9795,Double,49.7839,Double,49.98808949,String,1,String,10.1.0.46,String 63 | 10.2.0.38,01 24 2022 09 25 00,vegeta,String,http,String,1541407.051,Double,128.450587,Double,6349.9385,Double,49.354,Double,49.98610375,String,1,String,10.1.0.47,String 64 | 10.2.0.39,01 24 2022 09 25 00,vegeta,String,http,String,1779243.803,Double,148.270316,Double,14528.5116,Double,49.7856,Double,49.82562664,String,1,String,10.1.0.30,String 65 | 10.2.0.40,01 24 2022 09 25 00,vegeta,String,http,String,1672772.14,Double,139.397678,Double,3445.225936,Double,49.969201,Double,49.89826845,String,1,String,10.1.0.136,String 66 | 10.2.0.42,01 24 2022 09 25 00,vegeta,String,http,String,1596426.567,Double,133.035547,Double,6166.889733,Double,49.2802,Double,49.87551754,String,1,String,10.1.0.87,String 67 | 10.2.0.43,01 24 2022 09 25 00,vegeta,String,http,String,1613549.71,Double,134.462475,Double,4587.572204,Double,50.36584,Double,49.86678692,String,1,String,10.1.0.55,String 68 | 10.2.0.44,01 24 2022 09 25 00,vegeta,String,http,String,1355728.901,Double,112.977408,Double,1824.41779,Double,49.22425,Double,49.98198668,String,1,String,10.1.0.19,String 69 | 10.2.0.45,01 24 2022 09 25 00,vegeta,String,http,String,1466323.488,Double,122.193624,Double,2543.019,Double,49.410582,Double,49.9770804,String,1,String,10.1.0.57,String 70 | 10.2.0.47,01 24 2022 09 25 00,vegeta,String,http,String,1579140.452,Double,131.595037,Double,8908.2767,Double,49.736,Double,49.86724527,String,1,String,10.1.0.130,String 71 | 10.2.0.48,01 24 2022 09 25 00,vegeta,String,http,String,1674740.882,Double,139.56174,Double,5023.6072,Double,50.241754,Double,49.98576336,String,1,String,10.1.0.17,String 72 | 10.2.0.49,01 24 2022 09 25 00,vegeta,String,http,String,1587207.099,Double,132.267258,Double,10360.41623,Double,49.582293,Double,49.87807146,String,1,String,10.1.0.62,String 73 | 10.2.0.5,01 24 2022 09 25 00,vegeta,String,http,String,1969988.481,Double,164.165706,Double,9858.2126,Double,50.7931,Double,49.89097727,String,1,String,10.1.0.9,String 74 | 10.2.0.50,01 24 2022 09 25 00,vegeta,String,http,String,1603418.63,Double,133.618219,Double,4458.702115,Double,48.5004,Double,49.7630783,String,1,String,10.1.0.26,String 75 | 10.2.0.51,01 24 2022 09 25 00,vegeta,String,http,String,1733665.245,Double,144.472103,Double,8462.905731,Double,48.5355,Double,49.97686694,String,1,String,10.1.0.128,String 76 | 10.2.0.52,01 24 2022 09 25 00,vegeta,String,http,String,2324379.268,Double,193.698272,Double,15730.4552,Double,54.11175,Double,49.88379594,String,1,String,10.1.0.77,String 77 | 10.2.0.54,01 24 2022 09 25 00,vegeta,String,http,String,1629572.887,Double,135.79774,Double,3919.0512,Double,49.0456,Double,49.89293442,String,1,String,10.1.0.75,String 78 | 10.2.0.55,01 24 2022 09 25 00,vegeta,String,http,String,1660560.529,Double,138.380044,Double,5813.1267,Double,49.6436,Double,49.82972781,String,1,String,10.1.0.81,String 79 | 10.2.0.57,01 24 2022 09 25 00,vegeta,String,http,String,1643456.139,Double,136.954678,Double,12581.7151,Double,49.71622,Double,49.86542485,String,1,String,10.1.0.91,String 80 | 10.2.0.58,01 24 2022 09 25 00,vegeta,String,http,String,1460420.167,Double,121.70168,Double,4270.541603,Double,49.6036,Double,49.88404683,String,1,String,10.1.0.80,String 81 | 10.2.0.59,01 24 2022 09 25 00,vegeta,String,http,String,3006769.883,Double,250.564156,Double,5669.139466,Double,55.7521,Double,49.91411656,String,1,String,10.1.0.123,String 82 | 10.2.0.6,01 24 2022 09 25 00,vegeta,String,http,String,1654295.454,Double,137.857954,Double,3428.5949,Double,50.225642,Double,49.90319883,String,1,String,10.1.0.16,String 83 | 10.2.0.60,01 24 2022 09 25 00,vegeta,String,http,String,1733745.704,Double,144.478808,Double,7890.363,Double,48.455772,Double,49.86759372,String,1,String,10.1.0.58,String 84 | 10.2.0.61,01 24 2022 09 25 00,vegeta,String,http,String,1496706.503,Double,124.725541,Double,4751.305691,Double,49.5248,Double,49.87198403,String,1,String,10.1.0.131,String 85 | 10.2.0.62,01 24 2022 09 25 00,vegeta,String,http,String,1560920.923,Double,130.076743,Double,2742.0109,Double,49.7271,Double,49.97737108,String,1,String,10.1.0.132,String 86 | 10.2.0.63,01 24 2022 09 25 00,vegeta,String,http,String,1605602.918,Double,133.800243,Double,5274.9334,Double,50.0023,Double,49.98183768,String,1,String,10.1.0.68,String 87 | 10.2.0.65,01 24 2022 09 25 00,vegeta,String,http,String,1732941.737,Double,144.411811,Double,2975.024412,Double,49.769514,Double,49.88657401,String,1,String,10.1.0.34,String 88 | 10.2.0.66,01 24 2022 09 25 00,vegeta,String,http,String,1893243.35,Double,157.770279,Double,14159.22418,Double,49.5127,Double,49.98074923,String,1,String,10.1.0.76,String 89 | 10.2.0.67,01 24 2022 09 25 00,vegeta,String,http,String,1641712.815,Double,136.809401,Double,5366.264808,Double,49.661543,Double,49.82486245,String,1,String,10.1.0.117,String 90 | 10.2.0.68,01 24 2022 09 25 00,vegeta,String,http,String,1533720.168,Double,127.810014,Double,4815.849985,Double,48.638598,Double,49.97662837,String,1,String,10.1.0.8,String 91 | 10.2.0.69,01 24 2022 09 25 00,vegeta,String,http,String,1882972.133,Double,156.914344,Double,6235.7494,Double,50.1621,Double,49.8999055,String,1,String,10.1.0.15,String 92 | 10.2.0.7,01 24 2022 09 25 00,vegeta,String,http,String,1782483.115,Double,148.540259,Double,6261.2645,Double,49.970385,Double,49.90973073,String,1,String,10.1.0.60,String 93 | 10.2.0.70,01 24 2022 09 25 00,vegeta,String,http,String,1581041.014,Double,131.753417,Double,3000.7838,Double,49.612,Double,49.97786951,String,1,String,10.1.0.5,String 94 | 10.2.0.71,01 24 2022 09 25 00,vegeta,String,http,String,1429166.715,Double,119.097226,Double,9045.598409,Double,49.444797,Double,49.82030706,String,1,String,10.1.0.138,String 95 | 10.2.0.72,01 24 2022 09 25 00,vegeta,String,http,String,1441734.136,Double,120.144511,Double,4967.389673,Double,49.441771,Double,49.98288546,String,1,String,10.1.0.7,String 96 | 10.2.0.73,01 24 2022 09 25 00,vegeta,String,http,String,1695493.361,Double,141.291113,Double,14111.98472,Double,49.4579,Double,49.8296741,String,1,String,10.1.0.129,String 97 | 10.2.0.74,01 24 2022 09 25 00,vegeta,String,http,String,1697650.33,Double,141.47086,Double,19346.5013,Double,49.772639,Double,49.33208819,String,1,String,10.1.0.32,String 98 | 10.2.0.75,01 24 2022 09 25 00,vegeta,String,http,String,1816796.045,Double,151.39967,Double,10160.79601,Double,49.6061,Double,49.8345181,String,1,String,10.1.0.41,String 99 | 10.2.0.77,01 24 2022 09 25 00,vegeta,String,http,String,2435059.172,Double,202.921597,Double,10219.74144,Double,52.0843,Double,49.88461147,String,1,String,10.1.0.54,String 100 | 10.2.0.78,01 24 2022 09 25 00,vegeta,String,http,String,2405370.117,Double,200.447509,Double,4519.872508,Double,53.9703,Double,49.89668441,String,1,String,10.1.0.102,String 101 | 10.2.0.79,01 24 2022 09 25 00,vegeta,String,http,String,1399640.661,Double,116.636721,Double,6127.382167,Double,49.4216,Double,49.81937271,String,1,String,10.1.0.124,String 102 | 10.2.0.8,01 24 2022 09 25 00,vegeta,String,http,String,1773120.442,Double,147.760036,Double,7537.786732,Double,49.781376,Double,49.83665088,String,1,String,10.1.0.43,String 103 | 10.2.0.80,01 24 2022 09 25 00,vegeta,String,http,String,1708090.254,Double,142.340854,Double,2068.0354,Double,50.0938,Double,49.91606293,String,1,String,10.1.0.42,String 104 | 10.2.0.82,01 24 2022 09 25 00,vegeta,String,http,String,1396433.635,Double,116.369469,Double,2380.6837,Double,49.4918,Double,49.98107361,String,1,String,10.1.0.133,String 105 | 10.2.0.83,01 24 2022 09 25 00,vegeta,String,http,String,1603697.607,Double,133.641467,Double,9964.997933,Double,49.7571,Double,49.8318476,String,1,String,10.1.0.72,String 106 | 10.2.0.84,01 24 2022 09 25 00,vegeta,String,http,String,1612781.549,Double,134.398462,Double,1390.968734,Double,49.750927,Double,49.97839136,String,1,String,10.1.0.86,String 107 | 10.2.0.85,01 24 2022 09 25 00,vegeta,String,http,String,1661749.839,Double,138.479153,Double,5427.3287,Double,49.724198,Double,49.98817067,String,1,String,10.1.0.25,String 108 | 10.2.0.86,01 24 2022 09 25 00,vegeta,String,http,String,2214686.749,Double,184.557229,Double,2712.405548,Double,50.5172,Double,49.95856751,String,1,String,10.1.0.53,String 109 | 10.2.0.87,01 24 2022 09 25 00,vegeta,String,http,String,1902772.009,Double,158.564334,Double,8096.7735,Double,50.430639,Double,49.9776106,String,1,String,10.1.0.134,String 110 | 10.2.0.88,01 24 2022 09 25 00,vegeta,String,http,String,1643003.268,Double,136.916939,Double,11583.47104,Double,49.967275,Double,49.98690089,String,1,String,10.1.0.21,String 111 | 10.2.0.89,01 24 2022 09 25 00,vegeta,String,http,String,1500590.955,Double,125.049246,Double,4789.3954,Double,49.600695,Double,49.86864387,String,1,String,10.1.0.24,String 112 | 10.2.0.9,01 24 2022 09 25 00,vegeta,String,http,String,1452389.866,Double,121.032488,Double,3728.8017,Double,49.540265,Double,49.97888592,String,1,String,10.1.0.74,String 113 | 10.2.0.90,01 24 2022 09 25 00,vegeta,String,http,String,1822836.108,Double,151.903008,Double,3847.985406,Double,51.507912,Double,49.8969474,String,1,String,10.1.0.64,String 114 | 10.2.0.91,01 24 2022 09 25 00,vegeta,String,http,String,2741760.081,Double,228.480006,Double,12699.69533,Double,55.1685,Double,49.82748273,String,1,String,10.1.0.104,String 115 | 10.2.0.92,01 24 2022 09 25 00,vegeta,String,http,String,2247226.905,Double,187.268908,Double,4866.2415,Double,51.0948,Double,49.9031179,String,1,String,10.1.0.79,String 116 | 10.2.0.93,01 24 2022 09 25 00,vegeta,String,http,String,1718376.581,Double,143.198048,Double,8047.916647,Double,49.57497,Double,49.87130259,String,1,String,10.1.0.122,String 117 | 10.2.0.94,01 24 2022 09 25 00,vegeta,String,http,String,1844596.309,Double,153.716359,Double,4261.9191,Double,49.5554,Double,49.97744575,String,1,String,10.1.0.92,String 118 | 10.2.0.95,01 24 2022 09 25 00,vegeta,String,http,String,1910163.247,Double,159.18027,Double,9067.802907,Double,50.266391,Double,49.82167073,String,1,String,10.1.0.10,String 119 | 10.2.0.96,01 24 2022 09 25 00,vegeta,String,http,String,1852119.923,Double,154.343326,Double,10845.69276,Double,49.497,Double,49.98640431,String,1,String,10.1.0.22,String 120 | 10.2.0.97,01 24 2022 09 25 00,vegeta,String,http,String,1599739.128,Double,133.311594,Double,13925.61619,Double,49.585037,Double,49.86845686,String,1,String,10.1.0.49,String 121 | 10.2.0.98,01 24 2022 09 25 00,vegeta,String,http,String,1844790.269,Double,153.732522,Double,7550.1134,Double,48.831267,Double,49.87788909,String,1,String,10.1.0.84,String 122 | 10.2.0.99,01 24 2022 09 25 00,vegeta,String,http,String,1865902.289,Double,155.491857,Double,6968.526777,Double,50.9272,Double,49.83633075,String,1,String,10.1.0.39,String 123 | -------------------------------------------------------------------------------- /supplementals/stats/stats-IDPS-Off-D4-v4-vegeta-https-rate-50-1M-reply-120-instances-5-min.csv: -------------------------------------------------------------------------------- 1 | PartitionKey,RowKey,test_type,test_type@type,protocol,protocol@type,latency_total,latency_total@type,latency_mean,latency_mean@type,latency_max,latency_max@type,latency_min,latency_min@type,throughput,throughput@type,success,success@type,server,server@type 2 | 10.2.0.10,01 24 2022 14 30 00,vegeta,String,https,String,1225604.2959,Double,102.133691,Double,8255.9181,Double,50.4151,Double,49.41610771317947,String,1,String,10.1.0.72,String 3 | 10.2.0.10,01 24 2022 14 35 00,vegeta,String,https,String,1736365.247989,Double,144.697103,Double,30000.668134,Double,50.238995,Double,49.547411485890294,String,0.9998333333333334,String,10.1.0.72,String 4 | 10.2.0.100,01 24 2022 14 35 00,vegeta,String,https,String,2328627.965663,Double,194.05233,Double,30008.885178,Double,50.357831,Double,49.660480133490765,String,0.9999166666666667,String,10.1.0.73,String 5 | 10.2.0.102,01 24 2022 14 30 00,vegeta,String,https,String,1241812.552382,Double,103.484379,Double,9972.196261,Double,50.121968,Double,49.53136076720717,String,1,String,10.1.0.130,String 6 | 10.2.0.102,01 24 2022 14 35 00,vegeta,String,https,String,8431060.962911,Double,702.588413,Double,30001.48028,Double,54.796159,Double,49.58704642205735,String,0.9999166666666667,String,10.1.0.130,String 7 | 10.2.0.103,01 24 2022 14 30 00,vegeta,String,https,String,1305669.321061,Double,108.805776,Double,10713.122,Double,50.247662,Double,49.61509298404132,String,1,String,10.1.0.131,String 8 | 10.2.0.103,01 24 2022 14 35 00,vegeta,String,https,String,2333872.062853,Double,194.489338,Double,30000.649559,Double,50.227171,Double,49.564912157386644,String,0.99975,String,10.1.0.131,String 9 | 10.2.0.104,01 24 2022 14 30 00,vegeta,String,https,String,1275887.601227,Double,106.323966,Double,4629.122888,Double,50.1273,Double,49.611568111880764,String,1,String,10.1.0.132,String 10 | 10.2.0.104,01 24 2022 14 35 00,vegeta,String,https,String,1683646.110352,Double,140.303842,Double,8917.991196,Double,49.968822,Double,49.86194398738424,String,1,String,10.1.0.132,String 11 | 10.2.0.105,01 24 2022 14 35 00,vegeta,String,https,String,23430794.328862,Double,1952.566194,Double,30000.94815,Double,52.690005,Double,49.55148638127354,String,0.9995833333333334,String,10.1.0.133,String 12 | 10.2.0.106,01 24 2022 14 30 00,vegeta,String,https,String,1278322.713123,Double,106.526892,Double,30000.2851,Double,50.2302,Double,49.981786478301906,String,0.9999166666666667,String,10.1.0.112,String 13 | 10.2.0.106,01 24 2022 14 35 00,vegeta,String,https,String,1803014.221894,Double,150.251185,Double,30001.161253,Double,50.950068,Double,49.94424058950746,String,0.9998333333333334,String,10.1.0.112,String 14 | 10.2.0.107,01 24 2022 14 30 00,vegeta,String,https,String,1245416.664013,Double,103.784722,Double,7837.417027,Double,49.961884,Double,49.52179993056753,String,1,String,10.1.0.124,String 15 | 10.2.0.107,01 24 2022 14 35 00,vegeta,String,https,String,1620056.260031,Double,135.004688,Double,17205.234279,Double,50.113627,Double,49.679681635095946,String,1,String,10.1.0.124,String 16 | 10.2.0.108,01 24 2022 14 30 00,vegeta,String,https,String,1292502.502767,Double,107.708541,Double,2364.6703,Double,49.9698,Double,49.983358455247675,String,1,String,10.1.0.101,String 17 | 10.2.0.108,01 24 2022 14 35 00,vegeta,String,https,String,1853907.731924,Double,154.49231,Double,30002.412405,Double,50.049884,Double,49.666494937267814,String,0.9998333333333334,String,10.1.0.101,String 18 | 10.2.0.109,01 24 2022 14 30 00,vegeta,String,https,String,1180090.912396,Double,98.340909,Double,8173.798859,Double,49.216291,Double,49.43265786708408,String,1,String,10.1.0.65,String 19 | 10.2.0.109,01 24 2022 14 35 00,vegeta,String,https,String,1674351.303814,Double,139.529275,Double,30000.299193,Double,49.810324,Double,49.668532534540624,String,0.9999166666666667,String,10.1.0.65,String 20 | 10.2.0.11,01 24 2022 14 30 00,vegeta,String,https,String,1172657.018711,Double,97.721418,Double,1543.942735,Double,50.037757,Double,49.988326651186206,String,1,String,10.1.0.34,String 21 | 10.2.0.11,01 24 2022 14 35 00,vegeta,String,https,String,1877038.914657,Double,156.419909,Double,30000.415883,Double,50.175212,Double,49.547087557701204,String,0.9998333333333334,String,10.1.0.34,String 22 | 10.2.0.110,01 24 2022 14 30 00,vegeta,String,https,String,1225714.675751,Double,102.142889,Double,25701.606231,Double,50.435265,Double,49.2256469830165,String,1,String,10.1.0.137,String 23 | 10.2.0.110,01 24 2022 14 35 00,vegeta,String,https,String,1946819.379628,Double,162.234948,Double,30000.628536,Double,50.551726,Double,49.8581810229023,String,0.9998333333333334,String,10.1.0.137,String 24 | 10.2.0.111,01 24 2022 14 30 00,vegeta,String,https,String,1235486.2296,Double,102.957185,Double,3848.3796,Double,50.2612,Double,49.44189684015218,String,1,String,10.1.0.69,String 25 | 10.2.0.111,01 24 2022 14 35 00,vegeta,String,https,String,1943438.551745,Double,161.953212,Double,30003.395853,Double,50.584684,Double,49.69637095112347,String,0.9999166666666667,String,10.1.0.69,String 26 | 10.2.0.112,01 24 2022 14 30 00,vegeta,String,https,String,1149391.756775,Double,95.782646,Double,4851.989858,Double,50.3225,Double,49.43318457131913,String,1,String,10.1.0.129,String 27 | 10.2.0.112,01 24 2022 14 35 00,vegeta,String,https,String,2335720.118415,Double,194.643343,Double,30002.935376,Double,50.299843,Double,49.44378277267184,String,0.9999166666666667,String,10.1.0.129,String 28 | 10.2.0.113,01 24 2022 14 30 00,vegeta,String,https,String,1318221.359177,Double,109.851779,Double,26469.829839,Double,50.0209,Double,49.42531532430878,String,1,String,10.1.0.95,String 29 | 10.2.0.113,01 24 2022 14 35 00,vegeta,String,https,String,1907003.514882,Double,158.916959,Double,30003.164599,Double,50.017791,Double,49.864299688994954,String,0.9999166666666667,String,10.1.0.95,String 30 | 10.2.0.114,01 24 2022 14 30 00,vegeta,String,https,String,1302124.3301,Double,108.51036,Double,30000.272692,Double,50.1846,Double,49.61918838632737,String,0.9999166666666667,String,10.1.0.24,String 31 | 10.2.0.114,01 24 2022 14 35 00,vegeta,String,https,String,1758076.711935,Double,146.506392,Double,4199.503492,Double,51.088606,Double,49.86344916220636,String,1,String,10.1.0.24,String 32 | 10.2.0.115,01 24 2022 14 30 00,vegeta,String,https,String,1272198.3436,Double,106.016528,Double,5385.6292,Double,49.974,Double,49.98147482451981,String,1,String,10.1.0.104,String 33 | 10.2.0.115,01 24 2022 14 35 00,vegeta,String,https,String,1912088.673835,Double,159.340722,Double,30000.321846,Double,49.969313,Double,49.74990978326732,String,0.9999166666666667,String,10.1.0.104,String 34 | 10.2.0.116,01 24 2022 14 30 00,vegeta,String,https,String,1355937.621439,Double,112.994801,Double,15789.271704,Double,50.17104,Double,49.41890136658033,String,1,String,10.1.0.11,String 35 | 10.2.0.116,01 24 2022 14 35 00,vegeta,String,https,String,2275204.648358,Double,189.600387,Double,30000.907198,Double,50.08889,Double,49.07933115487825,String,0.9998333333333334,String,10.1.0.11,String 36 | 10.2.0.117,01 24 2022 14 30 00,vegeta,String,https,String,1184313.394854,Double,98.692782,Double,14222.33026,Double,50.474492,Double,49.522533894224374,String,1,String,10.1.0.123,String 37 | 10.2.0.117,01 24 2022 14 35 00,vegeta,String,https,String,1812617.453458,Double,151.051454,Double,30000.284469,Double,50.230229,Double,49.97864836230564,String,0.9999166666666667,String,10.1.0.123,String 38 | 10.2.0.118,01 24 2022 14 35 00,vegeta,String,https,String,2178751.201102,Double,181.5626,Double,30001.160987,Double,50.35627,Double,49.481868071375175,String,0.9999166666666667,String,10.1.0.55,String 39 | 10.2.0.119,01 24 2022 14 30 00,vegeta,String,https,String,1245615.3629,Double,103.80128,Double,4655.2051,Double,50.2684,Double,49.44066738849437,String,1,String,10.1.0.6,String 40 | 10.2.0.119,01 24 2022 14 35 00,vegeta,String,https,String,2334324.659852,Double,194.527054,Double,30000.235825,Double,50.404678,Double,49.64916562735274,String,0.9999166666666667,String,10.1.0.6,String 41 | 10.2.0.12,01 24 2022 14 30 00,vegeta,String,https,String,1426467.79417,Double,118.872316,Double,9742.573733,Double,49.882843,Double,49.42561108355966,String,1,String,10.1.0.134,String 42 | 10.2.0.12,01 24 2022 14 35 00,vegeta,String,https,String,1606600.542388,Double,133.883378,Double,16210.905115,Double,50.138749,Double,49.46169870649412,String,1,String,10.1.0.134,String 43 | 10.2.0.120,01 24 2022 14 30 00,vegeta,String,https,String,1317092.152378,Double,109.757679,Double,8289.606783,Double,50.1635,Double,49.22244229999123,String,1,String,10.1.0.18,String 44 | 10.2.0.120,01 24 2022 14 35 00,vegeta,String,https,String,2273076.649319,Double,189.423054,Double,21167.741863,Double,50.282314,Double,48.86315785756372,String,1,String,10.1.0.18,String 45 | 10.2.0.121,01 24 2022 14 30 00,vegeta,String,https,String,1412161.651351,Double,117.680137,Double,30004.983354,Double,50.792875,Double,49.42369375098726,String,0.9999166666666667,String,10.1.0.62,String 46 | 10.2.0.121,01 24 2022 14 35 00,vegeta,String,https,String,2146692.095752,Double,178.891007,Double,18329.933236,Double,50.843757,Double,49.58367696608377,String,1,String,10.1.0.62,String 47 | 10.2.0.122,01 24 2022 14 30 00,vegeta,String,https,String,1385871.924425,Double,115.489327,Double,17185.398982,Double,49.969374,Double,49.140815481809916,String,1,String,10.1.0.128,String 48 | 10.2.0.122,01 24 2022 14 35 00,vegeta,String,https,String,1791350.29812,Double,149.279191,Double,13902.985182,Double,50.28633,Double,49.74608369861773,String,1,String,10.1.0.128,String 49 | 10.2.0.123,01 24 2022 14 30 00,vegeta,String,https,String,1294951.980254,Double,107.912665,Double,30001.347637,Double,50.298528,Double,49.84196433766875,String,0.9998333333333334,String,10.1.0.67,String 50 | 10.2.0.123,01 24 2022 14 35 00,vegeta,String,https,String,1543101.725619,Double,128.59181,Double,18722.478943,Double,50.213587,Double,49.766491156329835,String,1,String,10.1.0.67,String 51 | 10.2.0.124,01 24 2022 14 35 00,vegeta,String,https,String,2581150.930137,Double,215.09591,Double,30002.318348,Double,51.548813,Double,49.74740558115309,String,0.9998333333333334,String,10.1.0.77,String 52 | 10.2.0.125,01 24 2022 14 30 00,vegeta,String,https,String,1433009.195893,Double,119.417432,Double,30000.742123,Double,50.192831,Double,49.33283748189984,String,0.9999166666666667,String,10.1.0.93,String 53 | 10.2.0.125,01 24 2022 14 35 00,vegeta,String,https,String,2311880.995594,Double,192.656749,Double,30000.657814,Double,50.17193,Double,49.15725558923707,String,0.9998333333333334,String,10.1.0.93,String 54 | 10.2.0.126,01 24 2022 14 30 00,vegeta,String,https,String,1439890.7332,Double,119.990894,Double,30000.5085,Double,50.3316,Double,49.80794717241023,String,0.9999166666666667,String,10.1.0.41,String 55 | 10.2.0.126,01 24 2022 14 35 00,vegeta,String,https,String,1901361.299745,Double,158.446774,Double,15260.354645,Double,50.326237,Double,49.67179351938898,String,1,String,10.1.0.41,String 56 | 10.2.0.127,01 24 2022 14 30 00,vegeta,String,https,String,1403621.313554,Double,116.968442,Double,19387.030804,Double,50.404,Double,49.14412335853165,String,1,String,10.1.0.125,String 57 | 10.2.0.127,01 24 2022 14 35 00,vegeta,String,https,String,2060550.4609,Double,171.712538,Double,30001.088052,Double,50.479429,Double,49.9729854263397,String,0.99975,String,10.1.0.125,String 58 | 10.2.0.128,01 24 2022 14 30 00,vegeta,String,https,String,1442451.313879,Double,120.204276,Double,20107.397718,Double,49.926213,Double,49.22323294964181,String,1,String,10.1.0.56,String 59 | 10.2.0.128,01 24 2022 14 35 00,vegeta,String,https,String,2206762.71814,Double,183.896893,Double,30000.374803,Double,50.076616,Double,49.65209904757205,String,0.9998333333333334,String,10.1.0.56,String 60 | 10.2.0.128,01 24 2022 14 40 00,vegeta,String,https,String,31724132.438418,Double,2643.677703,Double,30000.627731,Double,0.0511,Double,35.51956484101241,String,0.7103333333333334,String,10.1.0.56,String 61 | 10.2.0.129,01 24 2022 14 30 00,vegeta,String,https,String,1301728.1831,Double,108.477348,Double,10268.1294,Double,50.2462,Double,49.98821846424598,String,1,String,10.1.0.45,String 62 | 10.2.0.129,01 24 2022 14 35 00,vegeta,String,https,String,2175822.405541,Double,181.318533,Double,30000.215607,Double,50.164615,Double,49.47485209305895,String,0.9999166666666667,String,10.1.0.45,String 63 | 10.2.0.13,01 24 2022 14 30 00,vegeta,String,https,String,1323751.723539,Double,110.312643,Double,2479.635751,Double,49.830899,Double,49.98561227715639,String,1,String,10.1.0.30,String 64 | 10.2.0.13,01 24 2022 14 35 00,vegeta,String,https,String,1778200.147439,Double,148.183345,Double,21887.627523,Double,50.078795,Double,49.78224514134686,String,1,String,10.1.0.30,String 65 | 10.2.0.130,01 24 2022 14 30 00,vegeta,String,https,String,1318661.053154,Double,109.888421,Double,16191.701448,Double,50.2714,Double,49.58928102987804,String,1,String,10.1.0.42,String 66 | 10.2.0.130,01 24 2022 14 35 00,vegeta,String,https,String,2760321.785702,Double,230.026815,Double,30000.571025,Double,50.34805,Double,48.88362938367009,String,0.9999166666666667,String,10.1.0.42,String 67 | 10.2.0.131,01 24 2022 14 35 00,vegeta,String,https,String,1909666.421452,Double,159.138868,Double,15130.654923,Double,50.218228,Double,49.973479189325005,String,1,String,10.1.0.25,String 68 | 10.2.0.132,01 24 2022 14 30 00,vegeta,String,https,String,1345800.085436,Double,112.150007,Double,11306.483773,Double,49.918823,Double,49.231379716641634,String,1,String,10.1.0.118,String 69 | 10.2.0.132,01 24 2022 14 35 00,vegeta,String,https,String,2223218.213718,Double,185.268184,Double,16219.803936,Double,50.143483,Double,49.55300652548609,String,1,String,10.1.0.118,String 70 | 10.2.0.133,01 24 2022 14 35 00,vegeta,String,https,String,2533660.881648,Double,211.138406,Double,30002.027615,Double,50.254577,Double,49.45166873845963,String,0.9998333333333334,String,10.1.0.111,String 71 | 10.2.0.134,01 24 2022 14 30 00,vegeta,String,https,String,1351016.328066,Double,112.584694,Double,4797.9857,Double,50.2174,Double,49.430808612221064,String,1,String,10.1.0.5,String 72 | 10.2.0.134,01 24 2022 14 35 00,vegeta,String,https,String,1899798.930974,Double,158.316577,Double,30003.413858,Double,49.952967,Double,49.75199661402266,String,0.9999166666666667,String,10.1.0.5,String 73 | 10.2.0.135,01 24 2022 14 30 00,vegeta,String,https,String,1318690.658443,Double,109.890888,Double,8341.907918,Double,50.077531,Double,49.50860742954689,String,1,String,10.1.0.117,String 74 | 10.2.0.135,01 24 2022 14 35 00,vegeta,String,https,String,1931048.924137,Double,160.920743,Double,30000.249476,Double,49.862928,Double,48.96924670643575,String,0.9999166666666667,String,10.1.0.117,String 75 | 10.2.0.136,01 24 2022 14 30 00,vegeta,String,https,String,1372244.945402,Double,114.353745,Double,24474.6158,Double,50.4884,Double,49.43003003943698,String,1,String,10.1.0.58,String 76 | 10.2.0.136,01 24 2022 14 35 00,vegeta,String,https,String,1947727.676393,Double,162.310639,Double,30000.873185,Double,50.594579,Double,49.77149537067782,String,0.9998333333333334,String,10.1.0.58,String 77 | 10.2.0.137,01 24 2022 14 30 00,vegeta,String,https,String,1586538.860072,Double,132.211571,Double,30000.6117,Double,49.986999,Double,49.13940872347516,String,0.9998333333333334,String,10.1.0.127,String 78 | 10.2.0.137,01 24 2022 14 35 00,vegeta,String,https,String,1741409.397615,Double,145.117449,Double,18337.62705,Double,50.316235,Double,49.35877841800523,String,1,String,10.1.0.127,String 79 | 10.2.0.138,01 24 2022 14 30 00,vegeta,String,https,String,1262700.51585,Double,105.225042,Double,2026.415588,Double,49.822322,Double,49.98125644779997,String,1,String,10.1.0.108,String 80 | 10.2.0.138,01 24 2022 14 35 00,vegeta,String,https,String,16701858.674486,Double,1391.821556,Double,30003.899665,Double,53.465099,Double,49.42517646852718,String,0.9995,String,10.1.0.108,String 81 | 10.2.0.139,01 24 2022 14 30 00,vegeta,String,https,String,1298895.2162,Double,108.241268,Double,4464.6004,Double,50.5603,Double,49.51056949533387,String,1,String,10.1.0.122,String 82 | 10.2.0.139,01 24 2022 14 35 00,vegeta,String,https,String,2864106.425813,Double,238.675535,Double,30001.044247,Double,50.936733,Double,49.64562185153827,String,0.99975,String,10.1.0.122,String 83 | 10.2.0.14,01 24 2022 14 30 00,vegeta,String,https,String,1447355.478325,Double,120.612956,Double,30000.4712,Double,49.8328,Double,49.49876300937431,String,0.9999166666666667,String,10.1.0.29,String 84 | 10.2.0.14,01 24 2022 14 35 00,vegeta,String,https,String,2064802.949805,Double,172.066912,Double,14635.836229,Double,49.884848,Double,49.58110309739621,String,1,String,10.1.0.29,String 85 | 10.2.0.15,01 24 2022 14 30 00,vegeta,String,https,String,1260373.412679,Double,105.031117,Double,8185.151409,Double,49.9532,Double,49.42640831110625,String,1,String,10.1.0.91,String 86 | 10.2.0.15,01 24 2022 14 35 00,vegeta,String,https,String,1853716.865067,Double,154.476405,Double,30000.791905,Double,50.366405,Double,49.66495041252087,String,0.9999166666666667,String,10.1.0.91,String 87 | 10.2.0.16,01 24 2022 14 30 00,vegeta,String,https,String,1210881.63372,Double,100.906802,Double,4656.239094,Double,48.9921,Double,49.61485200720449,String,1,String,10.1.0.27,String 88 | 10.2.0.16,01 24 2022 14 35 00,vegeta,String,https,String,2462378.328629,Double,205.198194,Double,30001.520618,Double,51.005366,Double,49.89555272112627,String,0.9999166666666667,String,10.1.0.27,String 89 | 10.2.0.17,01 24 2022 14 35 00,vegeta,String,https,String,1617272.7654,Double,134.77273,Double,30000.258788,Double,50.64249,Double,49.840203818893286,String,0.9999166666666667,String,10.1.0.19,String 90 | 10.2.0.18,01 24 2022 14 35 00,vegeta,String,https,String,1607934.995303,Double,133.994582,Double,30001.155601,Double,50.084978,Double,49.754009115588325,String,0.9998333333333334,String,10.1.0.28,String 91 | 10.2.0.19,01 24 2022 14 35 00,vegeta,String,https,String,1741255.207395,Double,145.1046,Double,10716.77762,Double,50.186075,Double,49.791007072354915,String,1,String,10.1.0.17,String 92 | 10.2.0.20,01 24 2022 14 30 00,vegeta,String,https,String,1411916.556,Double,117.659713,Double,30000.6364,Double,50.1387,Double,49.53167859805299,String,0.9999166666666667,String,10.1.0.44,String 93 | 10.2.0.20,01 24 2022 14 35 00,vegeta,String,https,String,1893071.371548,Double,157.755947,Double,15700.649889,Double,50.317441,Double,49.779042280290106,String,1,String,10.1.0.44,String 94 | 10.2.0.21,01 24 2022 14 35 00,vegeta,String,https,String,1765859.09398,Double,147.154924,Double,24394.944086,Double,50.217827,Double,48.97581295456191,String,1,String,10.1.0.96,String 95 | 10.2.0.22,01 24 2022 14 35 00,vegeta,String,https,String,1583134.239162,Double,131.927853,Double,30002.479997,Double,50.238537,Double,49.88141779821225,String,0.9999166666666667,String,10.1.0.8,String 96 | 10.2.0.23,01 24 2022 14 30 00,vegeta,String,https,String,1565110.2093,Double,130.42585,Double,9342.6359,Double,50.308,Double,49.70981950856663,String,1,String,10.1.0.139,String 97 | 10.2.0.23,01 24 2022 14 35 00,vegeta,String,https,String,2120204.127513,Double,176.683677,Double,30000.483474,Double,50.260096,Double,49.45155450090803,String,0.9999166666666667,String,10.1.0.139,String 98 | 10.2.0.24,01 24 2022 14 30 00,vegeta,String,https,String,1331203.0719,Double,110.933589,Double,14781.525,Double,50.0452,Double,49.528488173482344,String,1,String,10.1.0.46,String 99 | 10.2.0.24,01 24 2022 14 35 00,vegeta,String,https,String,2188088.680185,Double,182.340723,Double,25197.027247,Double,50.291678,Double,49.769151005343424,String,1,String,10.1.0.46,String 100 | 10.2.0.25,01 24 2022 14 35 00,vegeta,String,https,String,24048637.989557,Double,2004.053165,Double,30001.15756,Double,56.450222,Double,49.34410676164597,String,0.9995,String,10.1.0.109,String 101 | 10.2.0.26,01 24 2022 14 30 00,vegeta,String,https,String,1282949.747736,Double,106.912478,Double,12615.122159,Double,50.148491,Double,49.61003732869629,String,1,String,10.1.0.114,String 102 | 10.2.0.26,01 24 2022 14 35 00,vegeta,String,https,String,2091249.179331,Double,174.270764,Double,24359.693983,Double,50.348068,Double,49.372650731732705,String,1,String,10.1.0.114,String 103 | 10.2.0.27,01 24 2022 14 35 00,vegeta,String,https,String,1739029.873446,Double,144.919156,Double,30000.53866,Double,50.281961,Double,49.66808985913108,String,0.9999166666666667,String,10.1.0.121,String 104 | 10.2.0.28,01 24 2022 14 35 00,vegeta,String,https,String,1804381.509723,Double,150.365125,Double,30000.620147,Double,50.089359,Double,49.88041444525872,String,0.9998333333333334,String,10.1.0.33,String 105 | 10.2.0.29,01 24 2022 14 30 00,vegeta,String,https,String,1427364.826443,Double,118.947068,Double,12283.594882,Double,49.745007,Double,49.13980976679498,String,1,String,10.1.0.39,String 106 | 10.2.0.29,01 24 2022 14 35 00,vegeta,String,https,String,2397062.969553,Double,199.755247,Double,30001.542153,Double,50.189539,Double,49.56583387456042,String,0.9996666666666667,String,10.1.0.39,String 107 | 10.2.0.30,01 24 2022 14 30 00,vegeta,String,https,String,1306005.964501,Double,108.83383,Double,4127.17124,Double,50.139044,Double,49.98375698946424,String,1,String,10.1.0.54,String 108 | 10.2.0.30,01 24 2022 14 35 00,vegeta,String,https,String,1927951.564929,Double,160.66263,Double,30001.926875,Double,50.428466,Double,49.54501102392737,String,0.99975,String,10.1.0.54,String 109 | 10.2.0.31,01 24 2022 14 30 00,vegeta,String,https,String,1390482.714033,Double,115.873559,Double,30000.546992,Double,50.096669,Double,49.746846064847475,String,0.9999166666666667,String,10.1.0.15,String 110 | 10.2.0.31,01 24 2022 14 35 00,vegeta,String,https,String,2051557.802922,Double,170.96315,Double,30000.484224,Double,50.136745,Double,49.30548451724431,String,0.9999166666666667,String,10.1.0.15,String 111 | 10.2.0.31,01 24 2022 14 40 00,vegeta,String,https,String,31854452.253013,Double,2654.537687,Double,30000.584995,Double,0.055201,Double,35.68627790224561,String,0.7136666666666667,String,10.1.0.15,String 112 | 10.2.0.32,01 24 2022 14 30 00,vegeta,String,https,String,1400067.814907,Double,116.672317,Double,6167.043276,Double,50.2409,Double,49.98472980354437,String,1,String,10.1.0.37,String 113 | 10.2.0.32,01 24 2022 14 35 00,vegeta,String,https,String,2252116.719848,Double,187.676393,Double,30002.060816,Double,50.738027,Double,48.371400155751395,String,0.9999166666666667,String,10.1.0.37,String 114 | 10.2.0.33,01 24 2022 14 30 00,vegeta,String,https,String,1310410.269138,Double,109.200855,Double,30000.456399,Double,50.336348,Double,49.115838144673646,String,0.9999166666666667,String,10.1.0.48,String 115 | 10.2.0.33,01 24 2022 14 35 00,vegeta,String,https,String,1876386.614961,Double,156.365551,Double,25582.521929,Double,50.242995,Double,49.85154011345679,String,1,String,10.1.0.48,String 116 | 10.2.0.35,01 24 2022 14 30 00,vegeta,String,https,String,1284170.033209,Double,107.014169,Double,13035.712442,Double,50.347959,Double,49.42872709935748,String,1,String,10.1.0.94,String 117 | 10.2.0.35,01 24 2022 14 35 00,vegeta,String,https,String,2539343.991459,Double,211.611999,Double,30000.702566,Double,50.410358,Double,48.76665903101043,String,0.9998333333333334,String,10.1.0.94,String 118 | 10.2.0.36,01 24 2022 14 35 00,vegeta,String,https,String,1893295.540814,Double,157.774628,Double,30001.58623,Double,50.144513,Double,49.87376879538183,String,0.9999166666666667,String,10.1.0.61,String 119 | 10.2.0.37,01 24 2022 14 30 00,vegeta,String,https,String,1220821.081132,Double,101.73509,Double,30000.888671,Double,49.655604,Double,49.98192456787396,String,0.9999166666666667,String,10.1.0.135,String 120 | 10.2.0.37,01 24 2022 14 35 00,vegeta,String,https,String,5427039.873943,Double,452.253322,Double,30001.890656,Double,53.587185,Double,49.6847708860043,String,0.99975,String,10.1.0.135,String 121 | 10.2.0.38,01 24 2022 14 30 00,vegeta,String,https,String,1502794.746815,Double,125.232895,Double,14195.858089,Double,50.322,Double,48.366249738508515,String,1,String,10.1.0.13,String 122 | 10.2.0.38,01 24 2022 14 35 00,vegeta,String,https,String,1894826.379818,Double,157.902198,Double,30001.091478,Double,50.435449,Double,49.75112115395713,String,0.9999166666666667,String,10.1.0.13,String 123 | 10.2.0.39,01 24 2022 14 30 00,vegeta,String,https,String,1279756.731074,Double,106.646394,Double,23587.610948,Double,50.1016,Double,49.217675255042835,String,1,String,10.1.0.21,String 124 | 10.2.0.39,01 24 2022 14 35 00,vegeta,String,https,String,1966436.854558,Double,163.869737,Double,30000.753924,Double,50.205639,Double,49.35142214480836,String,0.9998333333333334,String,10.1.0.21,String 125 | 10.2.0.40,01 24 2022 14 35 00,vegeta,String,https,String,1900417.438952,Double,158.368119,Double,10009.234711,Double,50.009617,Double,49.76118296383534,String,1,String,10.1.0.36,String 126 | 10.2.0.41,01 24 2022 14 30 00,vegeta,String,https,String,1261635.245085,Double,105.13627,Double,11550.520942,Double,49.925861,Double,49.98739398778238,String,1,String,10.1.0.99,String 127 | 10.2.0.41,01 24 2022 14 35 00,vegeta,String,https,String,2251473.452578,Double,187.622787,Double,30000.483734,Double,51.130581,Double,49.57514790020756,String,0.9998333333333334,String,10.1.0.99,String 128 | 10.2.0.42,01 24 2022 14 30 00,vegeta,String,https,String,1276427.331446,Double,106.368944,Double,10090.300334,Double,50.317296,Double,49.43301923052392,String,1,String,10.1.0.113,String 129 | 10.2.0.42,01 24 2022 14 35 00,vegeta,String,https,String,2269008.959406,Double,189.084079,Double,30001.028653,Double,50.262389,Double,49.47202469434257,String,0.9998333333333334,String,10.1.0.113,String 130 | 10.2.0.43,01 24 2022 14 30 00,vegeta,String,https,String,1323334.198728,Double,110.277849,Double,2583.149128,Double,49.8523,Double,49.9737951192422,String,1,String,10.1.0.26,String 131 | 10.2.0.43,01 24 2022 14 35 00,vegeta,String,https,String,2586800.24937,Double,215.566687,Double,30000.55263,Double,49.964582,Double,49.772966342400586,String,0.9998333333333334,String,10.1.0.26,String 132 | 10.2.0.44,01 24 2022 14 30 00,vegeta,String,https,String,1287703.170619,Double,107.308597,Double,5058.183471,Double,49.845967,Double,49.5080318559578,String,1,String,10.1.0.53,String 133 | 10.2.0.44,01 24 2022 14 35 00,vegeta,String,https,String,2808421.14966,Double,234.035095,Double,30001.222175,Double,49.984416,Double,49.07554217008989,String,0.9998333333333334,String,10.1.0.53,String 134 | 10.2.0.45,01 24 2022 14 30 00,vegeta,String,https,String,1455822.598151,Double,121.318549,Double,6164.950004,Double,50.267096,Double,49.22918505033782,String,1,String,10.1.0.82,String 135 | 10.2.0.45,01 24 2022 14 35 00,vegeta,String,https,String,2025690.655148,Double,168.807554,Double,30001.178616,Double,50.374035,Double,49.846458103202565,String,0.9999166666666667,String,10.1.0.82,String 136 | 10.2.0.46,01 24 2022 14 30 00,vegeta,String,https,String,1479155.325739,Double,123.262943,Double,30000.428212,Double,50.1278,Double,49.21565713749109,String,0.9999166666666667,String,10.1.0.86,String 137 | 10.2.0.46,01 24 2022 14 35 00,vegeta,String,https,String,1925415.318482,Double,160.451276,Double,30000.395186,Double,49.909251,Double,49.582331997358814,String,0.9999166666666667,String,10.1.0.86,String 138 | 10.2.0.47,01 24 2022 14 30 00,vegeta,String,https,String,1194913.1397,Double,99.576094,Double,3007.6563,Double,49.9417,Double,49.992252221580806,String,1,String,10.1.0.81,String 139 | 10.2.0.47,01 24 2022 14 35 00,vegeta,String,https,String,2281212.668709,Double,190.101055,Double,30002.74016,Double,49.885642,Double,49.57185520604726,String,0.99975,String,10.1.0.81,String 140 | 10.2.0.48,01 24 2022 14 30 00,vegeta,String,https,String,1456507.502393,Double,121.375625,Double,20483.809285,Double,50.3799,Double,49.1312386766496,String,1,String,10.1.0.83,String 141 | 10.2.0.48,01 24 2022 14 35 00,vegeta,String,https,String,1841748.054013,Double,153.479004,Double,30001.2969,Double,50.226321,Double,49.56657085305897,String,0.9998333333333334,String,10.1.0.83,String 142 | 10.2.0.5,01 24 2022 14 30 00,vegeta,String,https,String,1327323.58168,Double,110.610298,Double,8706.610131,Double,50.294895,Double,49.511756713861104,String,1,String,10.1.0.88,String 143 | 10.2.0.5,01 24 2022 14 35 00,vegeta,String,https,String,2034713.766945,Double,169.55948,Double,30001.349205,Double,50.200821,Double,49.884846191124204,String,0.99975,String,10.1.0.88,String 144 | 10.2.0.50,01 24 2022 14 30 00,vegeta,String,https,String,1213805.086213,Double,101.150423,Double,1828.611452,Double,50.177777,Double,49.97690135629545,String,1,String,10.1.0.126,String 145 | 10.2.0.50,01 24 2022 14 35 00,vegeta,String,https,String,1973444.062445,Double,164.453671,Double,16301.571063,Double,50.340425,Double,49.78465357932706,String,1,String,10.1.0.126,String 146 | 10.2.0.51,01 24 2022 14 30 00,vegeta,String,https,String,1317654.0708,Double,109.804505,Double,15752.4839,Double,50.32,Double,49.21412443845746,String,1,String,10.1.0.7,String 147 | 10.2.0.51,01 24 2022 14 35 00,vegeta,String,https,String,2021871.221852,Double,168.489268,Double,27561.055867,Double,50.414382,Double,49.679760043359906,String,1,String,10.1.0.7,String 148 | 10.2.0.52,01 24 2022 14 35 00,vegeta,String,https,String,2241209.812989,Double,186.767484,Double,30000.727018,Double,50.098646,Double,49.573971246804554,String,0.9998333333333334,String,10.1.0.138,String 149 | 10.2.0.53,01 24 2022 14 30 00,vegeta,String,https,String,1521542.940476,Double,126.795245,Double,30000.263858,Double,50.199664,Double,49.04220925107668,String,0.9999166666666667,String,10.1.0.60,String 150 | 10.2.0.53,01 24 2022 14 35 00,vegeta,String,https,String,2083284.115808,Double,173.607009,Double,30000.511982,Double,50.276163,Double,49.75701866563918,String,0.9999166666666667,String,10.1.0.60,String 151 | 10.2.0.54,01 24 2022 14 30 00,vegeta,String,https,String,1187116.355533,Double,98.926362,Double,1923.6907,Double,50.3463,Double,49.97698032590612,String,1,String,10.1.0.63,String 152 | 10.2.0.54,01 24 2022 14 35 00,vegeta,String,https,String,1696667.432952,Double,141.388952,Double,30000.369903,Double,50.489151,Double,49.95808052062368,String,0.9999166666666667,String,10.1.0.63,String 153 | 10.2.0.55,01 24 2022 14 30 00,vegeta,String,https,String,1293586.183,Double,107.798848,Double,4805.4347,Double,50.5239,Double,49.91552730839378,String,1,String,10.1.0.87,String 154 | 10.2.0.55,01 24 2022 14 35 00,vegeta,String,https,String,1594387.511578,Double,132.865625,Double,30000.772406,Double,50.494299,Double,49.47333367432438,String,0.9999166666666667,String,10.1.0.87,String 155 | 10.2.0.57,01 24 2022 14 35 00,vegeta,String,https,String,6266348.947018,Double,522.195745,Double,30003.431315,Double,53.992123,Double,49.25032276760312,String,0.99975,String,10.1.0.105,String 156 | 10.2.0.58,01 24 2022 14 30 00,vegeta,String,https,String,1285977.506019,Double,107.164792,Double,4415.872253,Double,49.801868,Double,49.990303690108234,String,1,String,10.1.0.89,String 157 | 10.2.0.58,01 24 2022 14 35 00,vegeta,String,https,String,1930605.17999,Double,160.883764,Double,30000.346029,Double,49.980262,Double,49.365741818869715,String,0.9999166666666667,String,10.1.0.89,String 158 | 10.2.0.6,01 24 2022 14 30 00,vegeta,String,https,String,1353464.278853,Double,112.788689,Double,2382.000832,Double,50.236065,Double,49.982397553688045,String,1,String,10.1.0.119,String 159 | 10.2.0.6,01 24 2022 14 35 00,vegeta,String,https,String,1636495.908252,Double,136.374659,Double,30000.322687,Double,49.984673,Double,49.886211044122504,String,0.9999166666666667,String,10.1.0.119,String 160 | 10.2.0.60,01 24 2022 14 30 00,vegeta,String,https,String,1488761.946,Double,124.063495,Double,30000.6853,Double,50.1948,Double,49.509276892047986,String,0.9999166666666667,String,10.1.0.23,String 161 | 10.2.0.60,01 24 2022 14 35 00,vegeta,String,https,String,1926955.850845,Double,160.579654,Double,30001.017933,Double,50.284554,Double,48.36710753859877,String,0.9999166666666667,String,10.1.0.23,String 162 | 10.2.0.62,01 24 2022 14 30 00,vegeta,String,https,String,1353706.1049,Double,112.808842,Double,28406.2227,Double,50.2513,Double,49.38959890665158,String,1,String,10.1.0.98,String 163 | 10.2.0.62,01 24 2022 14 35 00,vegeta,String,https,String,2006787.159838,Double,167.232263,Double,30000.300712,Double,50.247585,Double,49.894434089715496,String,0.9998333333333334,String,10.1.0.98,String 164 | 10.2.0.62,01 24 2022 14 40 00,vegeta,String,https,String,31667170.643484,Double,2638.930886,Double,30000.764701,Double,0.055901,Double,35.569606285951174,String,0.7113333333333334,String,10.1.0.98,String 165 | 10.2.0.63,01 24 2022 14 30 00,vegeta,String,https,String,1502931.709946,Double,125.244309,Double,30001.4351,Double,50.3095,Double,49.60785296948582,String,0.9999166666666667,String,10.1.0.66,String 166 | 10.2.0.63,01 24 2022 14 35 00,vegeta,String,https,String,2381013.114163,Double,198.417759,Double,30000.604409,Double,50.18182,Double,49.6833207292958,String,0.9998333333333334,String,10.1.0.66,String 167 | 10.2.0.65,01 24 2022 14 35 00,vegeta,String,https,String,2102791.372654,Double,175.232614,Double,21208.443444,Double,50.468189,Double,49.669439261971554,String,1,String,10.1.0.32,String 168 | 10.2.0.67,01 24 2022 14 30 00,vegeta,String,https,String,1257110.6302,Double,104.759219,Double,4185.8994,Double,50.035,Double,49.423499731533404,String,1,String,10.1.0.79,String 169 | 10.2.0.67,01 24 2022 14 35 00,vegeta,String,https,String,1610492.706336,Double,134.207725,Double,30000.260009,Double,50.211518,Double,49.57619261829873,String,0.9999166666666667,String,10.1.0.79,String 170 | 10.2.0.68,01 24 2022 14 30 00,vegeta,String,https,String,1228640.7324,Double,102.386727,Double,3857.8995,Double,50.2803,Double,49.9139732670742,String,1,String,10.1.0.92,String 171 | 10.2.0.68,01 24 2022 14 35 00,vegeta,String,https,String,1604099.851294,Double,133.674987,Double,30001.209285,Double,50.29871,Double,49.666844491708204,String,0.99975,String,10.1.0.92,String 172 | 10.2.0.69,01 24 2022 14 35 00,vegeta,String,https,String,2189173.444287,Double,182.43112,Double,30000.718144,Double,50.056318,Double,49.24281713027194,String,0.9999166666666667,String,10.1.0.90,String 173 | 10.2.0.7,01 24 2022 14 30 00,vegeta,String,https,String,1384556.460813,Double,115.379705,Double,30001.01308,Double,50.1871,Double,49.61937438328086,String,0.99975,String,10.1.0.136,String 174 | 10.2.0.7,01 24 2022 14 35 00,vegeta,String,https,String,1920262.238312,Double,160.021853,Double,30001.020573,Double,50.305989,Double,49.76079199484906,String,0.9998333333333334,String,10.1.0.136,String 175 | 10.2.0.70,01 24 2022 14 30 00,vegeta,String,https,String,1294608.205725,Double,107.884017,Double,4455.74632,Double,48.8762,Double,49.91376237138557,String,1,String,10.1.0.102,String 176 | 10.2.0.70,01 24 2022 14 35 00,vegeta,String,https,String,2523323.411743,Double,210.27695,Double,30000.29215,Double,54.403567,Double,49.55181779398072,String,0.9999166666666667,String,10.1.0.102,String 177 | 10.2.0.71,01 24 2022 14 30 00,vegeta,String,https,String,1361403.6158,Double,113.450301,Double,30000.4972,Double,50.4262,Double,49.358146624276216,String,0.9999166666666667,String,10.1.0.57,String 178 | 10.2.0.71,01 24 2022 14 35 00,vegeta,String,https,String,1961895.289579,Double,163.491274,Double,30000.899462,Double,50.305837,Double,49.97045600012392,String,0.9996666666666667,String,10.1.0.57,String 179 | 10.2.0.72,01 24 2022 14 30 00,vegeta,String,https,String,1270154.336693,Double,105.846194,Double,9031.468485,Double,50.006392,Double,49.42921368597201,String,1,String,10.1.0.40,String 180 | 10.2.0.72,01 24 2022 14 35 00,vegeta,String,https,String,1560972.976524,Double,130.081081,Double,10993.084402,Double,50.023221,Double,49.972954897709826,String,1,String,10.1.0.40,String 181 | 10.2.0.73,01 24 2022 14 30 00,vegeta,String,https,String,1206089.549735,Double,100.507462,Double,2981.6366,Double,50.1374,Double,49.73255097108051,String,1,String,10.1.0.49,String 182 | 10.2.0.73,01 24 2022 14 35 00,vegeta,String,https,String,2174256.179361,Double,181.188014,Double,30000.42138,Double,50.974669,Double,49.776747899326054,String,0.9999166666666667,String,10.1.0.49,String 183 | 10.2.0.74,01 24 2022 14 30 00,vegeta,String,https,String,1271583.923891,Double,105.965326,Double,30000.308837,Double,50.1033,Double,49.982520890420425,String,0.9999166666666667,String,10.1.0.64,String 184 | 10.2.0.74,01 24 2022 14 35 00,vegeta,String,https,String,2465640.003065,Double,205.47,Double,30000.534277,Double,50.442367,Double,49.64957113515484,String,0.9998333333333334,String,10.1.0.64,String 185 | 10.2.0.75,01 24 2022 14 30 00,vegeta,String,https,String,1358924.305483,Double,113.243692,Double,9037.882778,Double,49.9873,Double,49.229559395210686,String,1,String,10.1.0.20,String 186 | 10.2.0.75,01 24 2022 14 35 00,vegeta,String,https,String,1600078.547269,Double,133.339878,Double,15931.828273,Double,50.139927,Double,49.67177406915578,String,1,String,10.1.0.20,String 187 | 10.2.0.76,01 24 2022 14 30 00,vegeta,String,https,String,1337462.8473,Double,111.455237,Double,4671.4584,Double,50.0727,Double,49.50951376487452,String,1,String,10.1.0.51,String 188 | 10.2.0.76,01 24 2022 14 35 00,vegeta,String,https,String,2230381.566699,Double,185.86513,Double,30000.718805,Double,50.273217,Double,49.74570346305393,String,0.9998333333333334,String,10.1.0.51,String 189 | 10.2.0.77,01 24 2022 14 30 00,vegeta,String,https,String,1230331.221054,Double,102.527601,Double,1436.7646,Double,50.2124,Double,49.97673833359796,String,1,String,10.1.0.31,String 190 | 10.2.0.77,01 24 2022 14 35 00,vegeta,String,https,String,1734341.184438,Double,144.528432,Double,10541.009405,Double,50.208798,Double,49.86139127780311,String,1,String,10.1.0.31,String 191 | 10.2.0.78,01 24 2022 14 35 00,vegeta,String,https,String,1774521.111956,Double,147.876759,Double,30000.534257,Double,50.310003,Double,49.694905454430376,String,0.9999166666666667,String,10.1.0.78,String 192 | 10.2.0.79,01 24 2022 14 30 00,vegeta,String,https,String,1304517.1376,Double,108.709761,Double,4456.4357,Double,50.1256,Double,49.33177451613059,String,1,String,10.1.0.38,String 193 | 10.2.0.79,01 24 2022 14 35 00,vegeta,String,https,String,2091287.227961,Double,174.273935,Double,19697.842372,Double,50.337505,Double,49.36883606004198,String,1,String,10.1.0.38,String 194 | 10.2.0.8,01 24 2022 14 30 00,vegeta,String,https,String,1428589.652121,Double,119.049137,Double,26655.9983,Double,50.334,Double,49.51889571760127,String,1,String,10.1.0.76,String 195 | 10.2.0.8,01 24 2022 14 35 00,vegeta,String,https,String,1750331.318365,Double,145.860943,Double,30000.818998,Double,49.94928,Double,48.43014917463151,String,0.9998333333333334,String,10.1.0.76,String 196 | 10.2.0.80,01 24 2022 14 35 00,vegeta,String,https,String,2113678.127155,Double,176.139843,Double,24670.939984,Double,50.823113,Double,49.48541631906059,String,1,String,10.1.0.74,String 197 | 10.2.0.80,01 24 2022 14 40 00,vegeta,String,https,String,31764798.770965,Double,2647.066564,Double,30000.5155,Double,0.0531,Double,35.59037799619366,String,0.71175,String,10.1.0.74,String 198 | 10.2.0.81,01 24 2022 14 30 00,vegeta,String,https,String,1284443.665272,Double,107.036972,Double,7602.919919,Double,49.8126,Double,49.989398054543386,String,1,String,10.1.0.120,String 199 | 10.2.0.81,01 24 2022 14 35 00,vegeta,String,https,String,1935280.830602,Double,161.273402,Double,26293.59549,Double,50.003236,Double,49.8801386003571,String,1,String,10.1.0.120,String 200 | 10.2.0.82,01 24 2022 14 35 00,vegeta,String,https,String,2463423.274895,Double,205.285272,Double,30000.450617,Double,50.654669,Double,49.344222148167766,String,0.9996666666666667,String,10.1.0.115,String 201 | 10.2.0.83,01 24 2022 14 30 00,vegeta,String,https,String,1517200.53734,Double,126.433378,Double,17597.100928,Double,50.044083,Double,49.51774796417822,String,1,String,10.1.0.116,String 202 | 10.2.0.83,01 24 2022 14 35 00,vegeta,String,https,String,1911714.424228,Double,159.309535,Double,30000.475394,Double,50.254177,Double,49.67107218974338,String,0.9998333333333334,String,10.1.0.116,String 203 | 10.2.0.85,01 24 2022 14 30 00,vegeta,String,https,String,1403084.1022,Double,116.923675,Double,12703.9054,Double,49.9794,Double,49.219074011471356,String,1,String,10.1.0.71,String 204 | 10.2.0.85,01 24 2022 14 35 00,vegeta,String,https,String,1873652.735223,Double,156.137727,Double,11063.886717,Double,49.904998,Double,49.849539398812006,String,1,String,10.1.0.71,String 205 | 10.2.0.88,01 24 2022 14 30 00,vegeta,String,https,String,1361391.142832,Double,113.449261,Double,6805.486953,Double,49.986116,Double,49.147015479129315,String,1,String,10.1.0.70,String 206 | 10.2.0.88,01 24 2022 14 35 00,vegeta,String,https,String,2305339.237281,Double,192.111603,Double,30001.092174,Double,50.205095,Double,48.262365075263524,String,0.9999166666666667,String,10.1.0.70,String 207 | 10.2.0.90,01 24 2022 14 35 00,vegeta,String,https,String,1707855.666792,Double,142.321305,Double,30000.264936,Double,49.844797,Double,49.977365215057496,String,0.9999166666666667,String,10.1.0.16,String 208 | 10.2.0.92,01 24 2022 14 30 00,vegeta,String,https,String,1342938.595994,Double,111.911549,Double,30000.597017,Double,50.1418,Double,49.755240804150475,String,0.9999166666666667,String,10.1.0.106,String 209 | 10.2.0.92,01 24 2022 14 35 00,vegeta,String,https,String,2141207.375233,Double,178.433947,Double,30000.979245,Double,50.363955,Double,49.695511353033794,String,0.9999166666666667,String,10.1.0.106,String 210 | 10.2.0.93,01 24 2022 14 30 00,vegeta,String,https,String,1239889.8402,Double,103.324153,Double,4549.775311,Double,50.2165,Double,49.50855837589123,String,1,String,10.1.0.50,String 211 | 10.2.0.93,01 24 2022 14 35 00,vegeta,String,https,String,1733888.185568,Double,144.490682,Double,30000.354308,Double,49.951057,Double,49.76348761298586,String,0.9999166666666667,String,10.1.0.50,String 212 | 10.2.0.94,01 24 2022 14 30 00,vegeta,String,https,String,1453781.8646,Double,121.148488,Double,26677.5897,Double,50.8665,Double,49.143746633438354,String,1,String,10.1.0.12,String 213 | 10.2.0.94,01 24 2022 14 35 00,vegeta,String,https,String,1883873.541214,Double,156.989461,Double,30001.019198,Double,50.363427,Double,49.57801940065428,String,0.9998333333333334,String,10.1.0.12,String 214 | 10.2.0.95,01 24 2022 14 35 00,vegeta,String,https,String,5176212.733259,Double,431.351061,Double,23677.317434,Double,55.337569,Double,49.58227665406099,String,1,String,10.1.0.110,String 215 | 10.2.0.96,01 24 2022 14 30 00,vegeta,String,https,String,1294833.349208,Double,107.902779,Double,1439.6171,Double,49.9925,Double,49.97435300940557,String,1,String,10.1.0.80,String 216 | 10.2.0.96,01 24 2022 14 35 00,vegeta,String,https,String,1967857.583729,Double,163.988131,Double,28391.968962,Double,49.938784,Double,49.760489447139925,String,1,String,10.1.0.80,String 217 | 10.2.0.97,01 24 2022 14 35 00,vegeta,String,https,String,26375931.88375,Double,2197.994323,Double,30001.225549,Double,52.484462,Double,49.32872434458035,String,0.9994166666666666,String,10.1.0.107,String 218 | 10.2.0.98,01 24 2022 14 30 00,vegeta,String,https,String,1139153.3017,Double,94.929441,Double,30000.3008,Double,46.917,Double,49.977124500895435,String,0.9999166666666667,String,10.1.0.52,String 219 | 10.2.0.98,01 24 2022 14 35 00,vegeta,String,https,String,3074516.56289,Double,256.209713,Double,30001.089627,Double,51.036513,Double,49.878832411798435,String,0.99975,String,10.1.0.52,String 220 | -------------------------------------------------------------------------------- /supplementals/stats/stats-IDPS-on-off-table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/stats-IDPS-on-off-table.png -------------------------------------------------------------------------------- /supplementals/stats/stats-IDPS-on-off.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/stats-IDPS-on-off.png -------------------------------------------------------------------------------- /supplementals/stats/stats-rtt-IDPS-on-off.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/myers-dev/throughput-testing/efd03ec310d081660cbc3aad73534b04cd54142e/supplementals/stats/stats-rtt-IDPS-on-off.png -------------------------------------------------------------------------------- /supplementals/stats/~$IDPS-network-stats.xlsx: -------------------------------------------------------------------------------- 1 | Andriy Yerofyeyev Andriy Yerofyeyev -------------------------------------------------------------------------------- /supplementals/topology.drawio: -------------------------------------------------------------------------------- 1 | 7VxrU+I8FP41fLSTNr1+RATXGZ3Rdd+9fXEKDdC1ULYEQX/9m7QJtD3h4kqL7lZ3R3Kapul5zj0JLdyZrC4Tfza+iQMStQwUrFr4omUYuoUM9odTngUFGVZGGSVhkNHQhnAfvhDZUVAXYUDmgpaRaBxHNJwViYN4OiUDWqD5SRIvi92GcRQUCDN/RArT4IT7gR8R0O1bGNBxRnWtXO9PJByN5ZN1JK5MfNlZEOZjP4iXORLutnAniWOafZqsOiTi3JN8sb8kycB8nOBu59dNB0ff0dXvs2yw3mtuWb9CQqb0uEMLdJ/8aCH4Jd6VPksGLschJfczf8DbSyYlLXw+ppOItXT2MfDnYxKIxpwm8SPpxFGcpDdjhGy701lfkQgY4sZbn1KSTNO7DWQy6oGvKljyRBJKVjmgxatfknhCaPLMuoirZ1JshRxj8aLLjUx4tgB6nJMHW3b0hRyO1kNveM0+CHa/gvUYcJoETHJFcxpP2Z/zJF5Mg5S9iLXihI7jUTz1o+s4ngme/yKUPgu98xc0LsJDViH9nvv8gw+lWaJ1sRIjp41n2Ziy9/ueb2R3OZZsb+5LW5sbgzZX2c38GaUXcr6k14fsY044Ap+4w4FKbOyBS/pDpdi8TkDm8SIZkB39TGGS/GREdo0njQDHaKe8JSTyafhUtD5HFx6zCr3drZ1K/bbTHwjt0OK/nB5PaY6e/VSo59LtSDV3XKDn2FXouWFaFUFl7YeKv17IXNa13yfRbTwPaRhztvdjSuNJETfZtx2FI96HckNw7ovWgLGTJAApoY0lkBA67/Y42MyzzfhUJqsRjwI0/2WREO0pTOjCjx6mhC7j5JH1m8UhH777xJ4yV8lRXai60HqbKlBRRZjaANOADP1FRAG0KcvSx1vn7B+bUAe1LHalw1sa90oFQrntFAk6bPExioRy2ykS9PLweun5enmCOQJoFYZHpeej3ATZP+bBFjQKp0wuZbDH3QITuiAkGzshpDUn9Cpbxi2LcHu6IduC8Ugl1Mu5qY2YQ52lj7wapCoGrz7MyWCRhPT5YdP5PtUmOXDZCOqm0z1vQ/3q2j2j5+7V2YgMKR+WvV44HV2nrQuMoPFcP2etc6hCncNmQecsG2sWhsZUdsvrHa5K75x3HDPtDn22xVTGvpjqHcZM7sliJnHrLTeqG0m1kFt0D1YpZs9mKu4qieB6Gn8ulW4TjB3ktj0Lmg9HGYtVZT+8JharAdRaQzE5nSYWa2KxvzYWc4o6Z8hKVU7pDE+hdZUFYroOtA6omz+fZXI1DFecQ3khKqrihcpqhpO0sgtkS9AvwsmITT0K+/wF5gOfsL9fbx6uw+lipc2fRrXZQ6tYWTR0UzssPcW25laFDyzs6kjTNaSxefR0G4DFXpYWIUrIPHzx+2sd5nHyXOCgwqusWJMwCPjNCqzzuikMjNJHFiP5WtC091calOWjyhQNq4A0GiD3AlnUSs8GOKpCT70yhYQV25xCGmaD4xYcDdcrAGnZnqoOUS+YsKabU8oGzB1glnylbWreqcGExdz7GeMWDGgaFIVTNPWik3Q8AKGl1wmho4ZQbyDcBiEuZfHQPdaL4AE1tPkjoYOxhKn+5OIzySqjlzwVFRmGYsOD2fG8KtMNXHSHikzQqjM+hSW19s/ebaN4W+Az7SJ8ikyxTr0zYPUMQPdv1q6tUp3TwDqAquadBMYBRZemfP1qXE9cwDaUGT5qMvy9RVHbLCF54kzCgGnhp0W/wW8Lfm7JF6osbL2aCGMZmEGwccLZnLNsj18sO7Lh0BgoF+4Du29bdo3Bvm7AoFG5jcOoiNEYRh3Qk31IRrtascBhyv3zJ2M1jBna3CX3woQs/Si6X/SZUwbMb5ZKm6XSj7lUahmlzUCKqKDeTWtSt8sBnt5Ui/cEeLKut4YS1hlrDfAwDNWbvSaNAf27Deh6vft0BhSW94G+/SObTezyTiDTOf1mEwyL9wgA9AHDeQsUGEzNy/84J47tYb56PL4HFnEDU8V31+hju0q+g8KABy2QXHeopTBgwnz1eIWBkzK6FCubUKLrZTTMVo9XGHhHjDbNU0u0gq25kzSbQzPdDfWchUdPaw/39lM28kTMupGdiDHx/mPGrHVLkpDxgnvvgw7gFFemsEI8XIQcrBKPtoWQCbIe9cKWPDuENM+QJ4bE+SF7zwki1ii/1Ovkce/BHblrd+/JHbkseeSTOwwg/znXQQRmm5FLB3t0zyg6ApEs97b0d5D+pv6WUItt/XXP2tWffcjeUH234ZjK2a3ZkeEHjimBgTDOErj8THRHKy07VnzoyYTbpoBJkYkUyI/4L8xz1mu621eH32ig118zImopMmI6ePchliqUN9yuAy239KTHt9wHJEOvWbF//fnCAzFYC8h+L3k6Zioyl/XSK/uPzlqG7U/SYgBNXx21f/YAv/+4YCdKAJWX644Pmbha1A7d1OBiuqcAtLIKnXRcTakAnEux5JmQWgoFd5+vBl3c+TyLH7+E3tlXp31pninSqI26NYXwHce/ioVww4V5WlWF8Ivl77O77tXXm/+e+59+fJ1gQs4UQALsTr1bM13lfJDLnCrF2xaVVIWibZeybRuuTFkKFK0jJIFKFA/YTvbxIziMihVTS1G1qyzoULL9ABdVdwC3UzzeR/ymnCJcj/tHwrc3Anb66O1u4vQTd+qav14efyyHKzq5vlO4FQbXt2Yz+da4oHQYALvw285kRPxGAFlz802XWQ1h84WhuPs/7Vzbcto6FP2aPMZjS/LtMSHQnk47k2lm2p6+ZAwW4FODGSMC9OuPZEtgeYtL0tg0rUkyQUKW5b32TUsSV7g327zLo8X0UxbT9ArZ8eYK310h5Lg24v9EzVbW2MgtayZ5Epd19r7iIflJVUNZu0piupR1ZRXLspQlC71ylM3ndMS0uijPs7XebJylsVaxiCZUG4aoeBhFKQXNviYxm5a1gVtp/Z4mk6m6s2PLT2aRaiwrltMoztaVKty/wr08y1j5brbp0VRIT8nle74dPUzuP9yvv0TzUZxff/7Qvy47Gzznkt0j5HTOXrdrie5TlK6kvOSzsq0S4HqaMPqwiEaivOZacoVvp2yW8pLD38bRckpjWViyPPtBe1ma5cXF2LY9r9fbfaIQQPLC+4gxms+Lq5FNeK0cD80Z3dQAPPH0zg4Srsw0m1GWb/l1spdrpbZSj7F80PVeJ0JPAj2t6IOnGkZSDye7rvey5m+kuJ8hegwkTWOuubI4z+b8322ereZxIV6bl7KcTbNJNo/Sj1m2kDL/jzK2lXYXrVimw0M3CftWef+v6MpyZeluI3suCltVmPPn+1YtlFf5rirvrytK+wvjG2Gy+/HzmkEi5FJ8PuZvK8oRRzQYj0xq440COhwb1aaUmhDVC9SDiztb5SN6pB2RPirKJ/RYfxUXoOlbTtOIJU/66F5deUgTdnvcOo327RUvCO3YFT+iPpuzSn35atDOVdhRZu4HwM5xYLBzRNyGoHJPQyXEkPCQ9TEa0vQ+WyYsyYTYhxlj2UzHTbW9SZOJaMOEI7iNZGnEZUdzgJS0xhpItn3bHwiweWRbiKHMNhORBVjRz1VOrackZ6sofZxTts7yH7zdIktE9/0nfpelSY/aQjWA3puYQLUbwtQDmMZ0HK1SBqAtRFbc3r3lv3xAPfvK5Z/0RMkSUUmrqJd9vcKBJdGHXlEv+3qFU+/eqd3fqQ+wUgFKWvd27f52ZYD8l0ewFUuTOddLleyJsMCVLk7o3k9Iba0ovcmXCc8iw56DVFkK3jYp9XpJrAkPqIvilv+MChODnz4u6WiVJ2z7uG/8UFiT6rjuBB3i929voH31vQEaBCdtNqVjJrrlj5fMJx+L0h22ofPc3Wdnc3aDNoeJZnOuhy0XQ2eqmlXtDjdld/5vnDMdT30O5VToVE71FnKmoK2cSV56L5zqXlNdO9DDg1vL2cuByatqKrgbxsu1MuiSsbPCduhC9+Ebc7Gm/EfY5WItgNpqKqaG0+ViXS72x+Zivm5zSDFVFaNDocHqGkvEHOe0J42Wi1KvxslGSKiqRLop3pm8ZjIrmF2gW7L+LplN+NDTZCgeYDmKKP//5dPjx2S+2ljLp0lr/tDVmUXkEOu86Sn2rKApfCCx69iWY9kWH8fA8QBYXChMhyiny+RnNNzZsMiTlxIHE151w5olcSwuNmBdtU3pYIwxUs/kW0HTO800GOmjxgwNm4BEHZAngdStMvQAjqbU02nMICFjWzFIRDocD+CIglAD0vVCEw/RLpiQ060YZQfmETBrsdIjVnhpMCGZ+7Dg0oIJTYeiDIrE0YOkHwIIXadNCH0zhE4H4SEIcW0WD8NjuwiewaEtf1A2miqY2p9cfKYlEfpOTEXlDMOw4YH0wrDJ6QbWw6FhJui2mZ9CSu3m++C+M7wD8BFPh88wU2zT7hBkzwB0fyd37dZ4ToQdAFXLOwnQGaRLR18/G9cLE9gKRH0yYXcz/JOkqEdqSF54JoEgV/N+NezwO4BfUIuFJg/briUaOBqAHu8nWSyFyE7ExXogG4/RyLhwH3tDz/VaTPYdBJNG4zYO1JSgIX8CI9mbFHRg6QQHUfvnLyZqyG7ciJA8SHK6jtL0YTXkQRkIv1sq7ZZK3+ZSqYtqm4EMWUG7m9YQZKeKBM/p2OITCZ7i9XZQQp6x3QQPslTdXpPOgf7ZDnS33n05B3rGtr2/ZLOJV98JRPzLbzbBkES0AUBvMJ13AcFArLD68i+b26v5ciNyj10axMQk9wANsdek3AExEEIPpNYdWiEGFIKNEAMXFXQtVyZQo9sVNGTQXo8Y+I0ETcilNRrOOqonafaHZvr72lueHj3tItyvn7JRJ2J2hfJEDMGnjxnz0j3NEy4LEb3POoCjr0xhg3oEtu1jk3rcuLZNwKzHvLClzg7ZVojUiSF5fsg7cYKIF+oPVYLU3MEdtY335MkdHJj1+3kndzhA0bbSQCZm+55rB3ucEOmBQE6WBwfa+7bzS+1dgo62d0L3WHv+pnxC89XIJ8bR7cRRwgWOKYGOMC4ncNWROL5VW3Zs+NAThhwucClqIgXmR+IHznN2a7qHV4cPOmj59SGSI9lnQq+8+xAri6k67sCHnltF0tf33JDQBXJ/zor9888XvjxIljoDhX05YUJqbr/0yv/s6yvkRbOCDGDFo9s33wdA3i8m7CQF0Dhd9/qQyW5063CIBRfTQwOgjTF0+Ix9ZH8JVVA/l+KqMyEXJAogkVPZ6tAx4UfOf+lMOArgRK1VJpxAygeAd+n9msU656Na6DSZ3qG8pCkYPa823/bg2pRrgNFtahpIIIP0ByZx2NZJU9dA3LWbdxBIKL2ZJI4cEPblhAlJoy6JOwuy3zSJI3AfFgfsa7ep/GB6UDsUgAP4rWdqp/kvIsiL+2+8LLmE/ReH4v7/7V1bd6M4Ev41fgwHCUnAY27umXN6drOb3u2eecnBIDts28YLOJf+9SMwspFKduzEYJLgdB9bQsiivirVRSV54FzOnr6kweL+jyTi0wG2o6eBczXAGPkuEW9FzXNV47GqZpLG0arO3lTcxr941VDWLuOIZ1XdqipPkmkeL9TKMJnPeZgrdUGaJo9qs3EyjZSKRTDhyjCKitswmHLQ7Hsc5ferWo/WWv/G48m9/GZkV1dmgWxcVWT3QZQ81qqc64FzmSZJvvo0e7rk04J6ki5x6P7CT6Edjv717c4hvy+y+fezVWfDQ25ZP0LK5/lxu8arrh+C6bKiV/Ws+bMk4ON9nPPbRRAW5UfBJQPn4j6fTUUJiY9RkN3zqCpkeZr85JfJNEnLmx3bZuzycn1FIoCrG2+CPOfpvLwb20TU7vmoFUkeeJrzpxrQ1aN/4cmM5+mzaFJdPcN0dUvFx071oI8bnvBZBfR9jR+YbBhUfDhZd72htfhQkfsA0juA0jwSnFsV58lcvF2kyXIeleS1RSlJ8/tkksyD6dckWVQ0/x/P8+dK7oJlnqjw8Kc4/1H7/GfRlUWr0tVT1XNZeJaFuXi+H/XC6i6XyvLmvrK0uTE6L0R2M35RM4wLupTXx+JjjTmigHvj0MQ2LPT4aGxkm8MYJEuWach3tKsmszxIJ3xXf3ISKDDayW8pnwZ5/KDOPkdnHtKE3O6WTqN8s/IFoR3T4q+oT+Z5rX71alDOpdqRYu56QM4dzyDnmNCGoKIvQ1U8XixU1tdgxKc3SRbncVKQfZTkeTJTcZNtz6fxpGiTFxPBRVCVQkFOngKkKmnUQLLti+thAbbQbItiKLOnSWEFWMGvZcqthzjNl8H0bs7zxyT9Kdotkrjo/vpBfEtm4qO2UPXg7E1MoNoNYcoAphEfB8tpDqAtSVZ+Pb0Q/8SALu0BFVcui5JVaCWlQi+7agWCpaIPtUIvu2oF0rtH2vcjfYC1ClBSure177drAxT/hAZb5tN4LvhSGnuFWhBMF8V8M09U3FpjetNcVswsldpDWJYrwtsmpn7MiDURCnVRfuXvYSli8OpdxsNlGufPd5vGt6U0yY71SRAR9/riHMrXNRviofeizE75OC+6FY8Xzydfy9KVY8PJc/09a5mzG5Q5hygyR5ljUQdOprJZXe6cpuTO7bDNtNv02WZT4Zdsqg7aTN7JbKbq1ptiUt1wKrU9VT1QzWZfjbS6S2PB9TBez5Veb4ztpbZ9CqcP12iLNTV/+L0t1gKorZpicji9LdbbYh/WFnNVmcMyUlUTOuwbpK4xQwwhIHVA3IJsseKrcfxUUKjORKooXplmzXhWRnYBb1X1V/FsIoY+jUfFA2RhwMX7f/+4+xrPl09W9jBpbT6U1oYEBzOL7TUnOszymsIHBnaRbSHLtsQ4hogBsMTD5ipEKc/iX8FoLcOFnZxVOJjw0gVrFkdRcbMB67psVhOMUUeqlnwraLKXIw3G8FFjguaYgMQ9kC8Cqcb7fSiSJtMTNSaQMGR0uxCEgtNmD2AlegSpouj6AEKK2oTQNUOIegi3QehovgIUwnYR3MNTz37yPLyXMLVvwvybr+IvXwqDt7JjDMuq5NL3mzRqHAU4arA3aZtaEDru538Nb3rB2wIfYSp8huWSNuUOQx8dQPc5I2RUi6YIxxJA1fJ6Jd7DteuDZAfjeuIwmQRR9SPs3o94MfTCiIYkXARr1ZPA0CP8bTnq8duCn6fpQtMM264kwtwd6EGIfuJFxreEW2ug6opsPMahcXkwYiNGWYvGPsLQaDQuFuOmCA0zb6Ame5eE9iw1uEFsaOC1S2oY3TgvVPIwTvljMJ3eLkdCKQPi9wsy/YLM+1yQoVhLOTBYBe2mxmAYnSoNPFQaeFLp9AYCNPBkXG8NJYwztmvgwShVv6LdT6AfewJdr6qdbgLdIznokyxpMz3fgLiWIYTY7pK2A4OINgDoHZrzFAQYiOXXX+5pbXvpLzdC94hyLyImunt45LAm6Q4CAz6cgeS6QyuBAYlgI4GBkxJas5UJ5Oh2CQ0jaMcLDHSI0IScmqP32PskzSBg3RR/O1Zktq/tvJG8663IlSck57utZMeer/K3TI6rk91zId2lHByf7jAcA+h+yHrb4XsQ9sRgzSAv8/jpiAkd683Cifhvnw0wC2alKZ+Xj26f/zUE9H61u10Z8I0728eHrLqqSgciFlwK8w2ANuZfO9DQry2F9ZGSHVnIaqQEe1CRtxopIdAlAOCdOp+njIPfyUC4yYvbpvmagpExzR5jMHZJDTDSpswEAj2MD2gmOLbqVFODY9euZiPQ4eimmUD25/HTERM6FZ/ZTDgAso6aCQSu0wvAvvdJh9sXUtSkUceDZ2/ITMRjIxj8P/r+LT3PlqP/xO4/Jz/O59H8bA/roJMBXYOu2Q+t7em8DKuWm9PuHiUjPIgCON6+dX+9zd5ab6z/s35tyy57BQVzMmlth384DbIsDgf1Tf4wL3W1mhOkuXYwQFlXOxpgRQVtk/zrtuub6bxqV9+ub25HzDx09BONdo2yF1aMXE1YSQeEdQ9T8bPA42nw0A7AA43PTwuPr8HDOgCPjAhvObhmo+iuN7VHOMdGKsN1YXXmTOn+7j7HT5RueBqLpy/ALyvDZfqwzdUbeyE3L5+OPEroJq9B6kZysM41nqrTtOo0nA5oxneL63/YwTbiUYPnWoNKKjc9a+feMJ3Vq7M4hnu2px7SOHs1gg2fH3Y4zi4S9jMTxrYGl9uBmWmPFbpPAg+ialI1tZnlwshoy/jssZL3WfDRdvRT1AV84OLgp8XH1fDBXcDHsLOtkSMDm4s77BNCeFVsomnjyXBMYAfjDnscCfBZBNjTBNjpggC/16zcJjxbDR/SBXwM5+99VoCwrQFEOwEQxKOPPZw69vDWUAHw7bGzZT+F7GKlrsEZuLAj7dglyphl60t0Ww7UPTiA4ZkHvTWAobUXY9PEo4EABtr9Qx29/HwQ+dHDxo5FXilBjq6nfYs0JkHbhr1dhjAc3VGlyIya4dSDbh5Iud9uoBPkV+h55pT5phP52z3WEMYPO3o8ZYdhVacLzIjltwfrDpdcgfUf32Dm0wvJgU3Z2WdIy2BFlAB6yaRWZaMLa4peMAxY5v2KqluePsSh+HCxzLpCQELVmeTMsCOrXfp5gDRvtqBOmdIjf0HEtjxEBvVfEUGOP9j5OyKioFtvb7KUjpkg5DZieoGECqHXPEx8hnxSvJNX2mHC82UWtTcvrOYnIl9zkY73MyFm8vkflckpQzUmPxPK33c7x+VeI8xbJirT2m5tXZkjy0UUyavslbxMPHXzwPoUuJZ413CCp1Rw30rbTSi6PEmLeFxHdJyrHctv0nHIMRhVjSk5w27uDyL/NvZV+S9SGA+V/9oSGrOVqIfwEu2Xf3rUEPboiuLctjPovcw9rkU833URc7Dr2L56XsU6ua2tqQjGs9+3GNX43iEq3yN0Cr4/HT9rAQ4ZSDuYY/0XOPZIDCuKm5//XjXf/Iq6c/03 --------------------------------------------------------------------------------