├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── lwipopts.h └── main.c /.gitignore: -------------------------------------------------------------------------------- 1 | lwip 2 | dpdk 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROGS = app 2 | 3 | CC = gcc 4 | PKGCONF = pkg-config 5 | 6 | CLEANFILES = $(PROGS) *.o *.d 7 | 8 | DPDK_VER=22.03 9 | LWIP_VER=2.1.3 10 | CONTRIB_VER=2.1.0 11 | 12 | NO_MAN= 13 | CFLAGS = -O3 -pipe -g -rdynamic 14 | CFLAGS += -Werror -Wall -Wunused-function 15 | CFLAGS += -Wextra 16 | CFLAGS += -I. 17 | 18 | LDFLAGS += -lpthread -lm 19 | 20 | C_SRCS = main.c 21 | 22 | C_OBJS = $(C_SRCS:.c=.o) 23 | 24 | # for dpdk 25 | DPDK_DIR = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))dpdk 26 | DPDK_SRC_DIR = $(DPDK_DIR)/dpdk-$(DPDK_VER) 27 | DPDK_INSTALL_DIR = $(DPDK_DIR)/install 28 | DPDK_PKG_CONFIG_PATH=$(DPDK_INSTALL_DIR)/lib/x86_64-linux-gnu/pkgconfig 29 | DPDK_PKG_CONFIG_FILE=$(DPDK_PKG_CONFIG_PATH)/libdpdk.pc 30 | CFLAGS += $(shell PKG_CONFIG_PATH=$(DPDK_PKG_CONFIG_PATH) $(PKGCONF) --cflags libdpdk) 31 | LDFLAGS += $(shell PKG_CONFIG_PATH=$(DPDK_PKG_CONFIG_PATH) $(PKGCONF) --libs libdpdk) 32 | 33 | # for lwip 34 | LWIP_DIR = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))lwip 35 | LWIP_SRC_DIR = $(LWIP_DIR)/lwip-$(LWIP_VER) 36 | CONTRIB_SRC_DIR = $(LWIP_DIR)/contrib-$(CONTRIB_VER) 37 | CFLAGS += -I$(LWIP_SRC_DIR)/src/include -I$(CONTRIB_SRC_DIR) -I$(CONTRIB_SRC_DIR)/ports/unix/port/include 38 | LWIP_OBJS = $(LWIP_SRC_DIR)/src/api/api_lib.o \ 39 | $(LWIP_SRC_DIR)/src/api/api_msg.o \ 40 | $(LWIP_SRC_DIR)/src/api/err.o \ 41 | $(LWIP_SRC_DIR)/src/api/if_api.o \ 42 | $(LWIP_SRC_DIR)/src/api/netbuf.o \ 43 | $(LWIP_SRC_DIR)/src/api/netdb.o \ 44 | $(LWIP_SRC_DIR)/src/api/netifapi.o \ 45 | $(LWIP_SRC_DIR)/src/api/sockets.o \ 46 | $(LWIP_SRC_DIR)/src/api/tcpip.o \ 47 | $(LWIP_SRC_DIR)/src/core/altcp_alloc.o \ 48 | $(LWIP_SRC_DIR)/src/core/altcp.o \ 49 | $(LWIP_SRC_DIR)/src/core/altcp_tcp.o \ 50 | $(LWIP_SRC_DIR)/src/core/def.o \ 51 | $(LWIP_SRC_DIR)/src/core/dns.o \ 52 | $(LWIP_SRC_DIR)/src/core/inet_chksum.o \ 53 | $(LWIP_SRC_DIR)/src/core/init.o \ 54 | $(LWIP_SRC_DIR)/src/core/ip.o \ 55 | $(LWIP_SRC_DIR)/src/core/ipv4/autoip.o \ 56 | $(LWIP_SRC_DIR)/src/core/ipv4/dhcp.o \ 57 | $(LWIP_SRC_DIR)/src/core/ipv4/etharp.o \ 58 | $(LWIP_SRC_DIR)/src/core/ipv4/icmp.o \ 59 | $(LWIP_SRC_DIR)/src/core/ipv4/igmp.o \ 60 | $(LWIP_SRC_DIR)/src/core/ipv4/ip4_addr.o \ 61 | $(LWIP_SRC_DIR)/src/core/ipv4/ip4.o \ 62 | $(LWIP_SRC_DIR)/src/core/ipv4/ip4_frag.o \ 63 | $(LWIP_SRC_DIR)/src/core/ipv6/dhcp6.o \ 64 | $(LWIP_SRC_DIR)/src/core/ipv6/ethip6.o \ 65 | $(LWIP_SRC_DIR)/src/core/ipv6/icmp6.o \ 66 | $(LWIP_SRC_DIR)/src/core/ipv6/inet6.o \ 67 | $(LWIP_SRC_DIR)/src/core/ipv6/ip6_addr.o \ 68 | $(LWIP_SRC_DIR)/src/core/ipv6/ip6.o \ 69 | $(LWIP_SRC_DIR)/src/core/ipv6/ip6_frag.o \ 70 | $(LWIP_SRC_DIR)/src/core/ipv6/mld6.o \ 71 | $(LWIP_SRC_DIR)/src/core/ipv6/nd6.o \ 72 | $(LWIP_SRC_DIR)/src/core/mem.o \ 73 | $(LWIP_SRC_DIR)/src/core/memp.o \ 74 | $(LWIP_SRC_DIR)/src/core/netif.o \ 75 | $(LWIP_SRC_DIR)/src/core/pbuf.o \ 76 | $(LWIP_SRC_DIR)/src/core/raw.o \ 77 | $(LWIP_SRC_DIR)/src/core/stats.o \ 78 | $(LWIP_SRC_DIR)/src/core/sys.o \ 79 | $(LWIP_SRC_DIR)/src/core/tcp.o \ 80 | $(LWIP_SRC_DIR)/src/core/tcp_in.o \ 81 | $(LWIP_SRC_DIR)/src/core/tcp_out.o \ 82 | $(LWIP_SRC_DIR)/src/core/timeouts.o \ 83 | $(LWIP_SRC_DIR)/src/core/udp.o \ 84 | $(LWIP_SRC_DIR)/src/netif/ethernet.o \ 85 | $(CONTRIB_SRC_DIR)/ports/unix/port/sys_arch.o 86 | 87 | OBJS = $(C_OBJS) $(LWIP_OBJS) 88 | 89 | CLEANFILES += $(LWIP_OBJS) 90 | 91 | .PHONY: all 92 | all: $(PROGS) 93 | 94 | $(DPDK_SRC_DIR).tar.xz: 95 | wget -P $(DPDK_DIR) https://fast.dpdk.org/rel/dpdk-$(DPDK_VER).tar.xz 96 | 97 | $(CONTRIB_SRC_DIR).zip: 98 | wget -P $(LWIP_DIR) http://download.savannah.nongnu.org/releases/lwip/contrib-$(CONTRIB_VER).zip 99 | 100 | $(LWIP_SRC_DIR).zip: 101 | wget -P $(LWIP_DIR) http://download.savannah.nongnu.org/releases/lwip/lwip-$(LWIP_VER).zip 102 | 103 | $(DPDK_SRC_DIR): $(DPDK_SRC_DIR).tar.xz 104 | tar xvf $< -C $(DPDK_DIR) 105 | 106 | $(CONTRIB_SRC_DIR): $(CONTRIB_SRC_DIR).zip 107 | unzip -n $< -d $(LWIP_DIR) 108 | 109 | $(LWIP_SRC_DIR): $(LWIP_SRC_DIR).zip 110 | unzip -n $< -d $(LWIP_DIR) 111 | 112 | $(DPDK_PKG_CONFIG_FILE): $(DPDK_SRC_DIR) 113 | meson --prefix=$(DPDK_INSTALL_DIR) --libdir=lib/x86_64-linux-gnu $(DPDK_SRC_DIR)/build $(DPDK_SRC_DIR) 114 | ninja -C $(DPDK_SRC_DIR)/build 115 | ninja -C $(DPDK_SRC_DIR)/build install 116 | 117 | $(OBJS): $(CONTRIB_SRC_DIR) $(LWIP_SRC_DIR) $(DPDK_PKG_CONFIG_FILE) 118 | 119 | $(PROGS): $(OBJS) 120 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) 121 | 122 | clean: 123 | -@rm -rf $(CLEANFILES) 124 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Running lwIP on DPDK 2 | 3 | This repository contains an extremely lightweight web server application, built atop lwIP and DPDK. 4 | 5 | The primary objective of this repository is to run a quick benchmark to see the baseline performance of lwIP running on DPDK. 6 | 7 | We assume to use [wrk](https://github.com/wg/wrk) as the HTTP benchmark client, to measure the throughput and latency. 8 | 9 | While we use the HTTP benchmark client, this workload would be simple TCP ping-pong. 10 | 11 | ## Target platform 12 | 13 | This repository assumes Linux on x86-64 CPUs. 14 | 15 | The author tested this on Ubuntu 20.04. 16 | 17 | ## Requirements 18 | 19 | meson and ninja are used for DPDK compilation. ( Please see https://doc.dpdk.org/guides/linux_gsg/build_dpdk.html for details. ) 20 | 21 | Please install them if you do not have yet. 22 | 23 | - meson 24 | - ninja 25 | 26 | ## Build 27 | 28 | The following commands will compile the lwIP and DPDK applied application. 29 | 30 | ``` 31 | git clone https://github.com/yasukata/tinyhttpd-lwip-dpdk.git 32 | ``` 33 | 34 | ``` 35 | cd tinyhttpd-lwip-dpdk 36 | ``` 37 | 38 | ``` 39 | make 40 | ``` 41 | 42 | ### What Makefile does 43 | 44 | Makefile produces a file named app that is bound with lwIP and DPDK. 45 | 46 | Our Makefile downloads the source code of lwIP and DPDK, and compile them. The source code and the build objects of lwIP and DPDK are located in the directory where our Makefile is located. 47 | 48 | The following is the detailed procedure that our Makefile conducts. 49 | 50 | 1. download required the source code of lwIP and DPDK. 51 | 52 | - lwIP: ([LOCATION_OF_MAKEFILE]/lwip/lwip-$(LWIP_VER).zip) http://download.savannah.nongnu.org/releases/lwip/lwip-$(LWIP_VER).zip 53 | - lwIP contrib: ([LOCATION_OF_MAKEFILE]/lwip/contrib-$(CONTRIB_VER).zip) http://download.savannah.nongnu.org/releases/lwip/contrib-$(CONTRIB_VER).zip 54 | - DPDK: ([LOCATION_OF_MAKEFILE]/dpdk/dpdk-$(DPDK_VER).tar.xz) https://fast.dpdk.org/rel/dpdk-$(DPDK_VER).tar.xz 55 | 56 | 2. extract the source code 57 | 58 | - lwIP: ([LOCATION_OF_MAKEFILE]/lwip/lwip-$(LWIP_VER)) 59 | - lwIP contrib: ([LOCATION_OF_MAKEFILE]/lwip/contrib-$(CONTRIB_VER)) 60 | - DPDK: ([LOCATION_OF_MAKEFILE]/dpdk/dpdk-$(DPDK_VER)) 61 | 62 | 3. compile and install DPDK 63 | 64 | DPDK is installed in a directory [LOCATION_OF_MAKEFILE]/dpdk/install . 65 | 66 | Therefore, you do not need the root permission for installation, and this does not overwrite the existing DPDK library. 67 | 68 | ## How to use 69 | 70 | We assume the command is executed in the top directory of this repository. So, please cd to tinyhttpd-lwip-dpdk. 71 | 72 | ``` 73 | cd tinyhttpd-lwip-dpdk 74 | ``` 75 | 76 | ### Launch app with a tap device 77 | 78 | The following command launches the application. This example uses a tap device for the network interface so that users, who do not have an extra network interface for DPDK, also can try. 79 | 80 | **WARNING: the following command is executed with sudo. so, please conduct the following procedure only when you understand what you are doing.** 81 | 82 | Here, the root permission is necessary to execute DPDK. 83 | 84 | ``` 85 | sudo LD_LIBRARY_PATH=./dpdk/install/lib/x86_64-linux-gnu ./app -l 0-1 --proc-type=primary --file-prefix=pmd1 --vdev=net_tap001,iface=tap001 --no-pci -- -a 10.0.0.2 -g 10.0.0.1 -m 255.255.255.0 -l 1 -p 10000 86 | ``` 87 | 88 | In the command above, ```app [option for DPDK] -- [option for lwIP and app]``` 89 | 90 | Option for DPDK: 91 | - ```--vdev=net_tap001,iface=tap001```: the network interface to be used 92 | 93 | Please refer to https://doc.dpdk.org/guides/nics/tap.html for details. 94 | 95 | Option for lwIP and app: 96 | - ```-a```: IP address (10.0.0.2) 97 | - ```-g```: gateway address (10.0.0.1) 98 | - ```-m```: netmask (255.255.255.0) 99 | - ```-l```: content length that the web server app serves (1) 100 | - ```-p```: the port the web server app listens on (10000) 101 | 102 | ### It didn't work? 103 | 104 | If the app does not work, and if particularly failing at rte_eal_init, it would be due to lack of hugepages. 105 | 106 | Please type the following command to see how many hugepages are available on your machine. 107 | 108 | ``` 109 | cat /proc/sys/vm/nr_hugepages 110 | ``` 111 | 112 | If the output is 0, it means your machine has no hugepage. 113 | 114 | If you wish to request the OS to have some hugepages, you can do that by the following command; the example command below requests to preserve 64 MB of hugepages, and please change the hugepage size according to your machine's DRAM size. 115 | 116 | NOTE: if you are not familiar with hugepages, it would be recommended to begin by giving **sufficiantly small DRAM for hugepages**, compared to the entire DRAM that your machine has. 117 | 118 | **WARNING: the following command requires sudo. please type the following command only when you understand what you are doing.** 119 | 120 | ``` 121 | sudo ./dpdk/dpdk-22.03/usertools/dpdk-hugepages.py -p 2M -r 64M 122 | ``` 123 | 124 | After running the command above, if you try the same following command to see the configured hugepages, it will say 32; it means 32 of 2 MB hugepages (equal 64 MB). 125 | 126 | ``` 127 | cat /proc/sys/vm/nr_hugepages 128 | ``` 129 | 130 | You can find a detailed explanation in https://doc.dpdk.org/guides/linux_gsg/sys_reqs.html . 131 | 132 | ### Configure a bridge for the tap device 133 | 134 | Once you launch the app by the command above, DPDK polls the tap device named tap001, and lwIP waits for the packets for 10.0.0.2. 135 | 136 | In this example, we send packets to tap001 through a bridge. 137 | 138 | The following command does: 139 | - create a bridge named br001 140 | - configure the bridge address as 10.0.0.1/24 141 | - turn on the bridge 142 | - finally, attache tap001 to br001 143 | 144 | ``` 145 | sudo brctl addbr br001; sudo ifconfig br001 10.0.0.1 netmask 255.255.255.0 up; sudo brctl addif br001 tap001 146 | ``` 147 | 148 | After you do this, please try ping first to confirm the reachability to lwIP. Supposedly, you fill receive pong from lwIP (10.0.0.2). 149 | 150 | ``` 151 | ping 10.0.0.2 152 | ``` 153 | 154 | ### Benchmark client 155 | 156 | For the benchmark client, we recommend to use [wrk](https://github.com/wg/wrk). 157 | 158 | Please visit https://github.com/wg/wrk for details. 159 | 160 | ### Launch app with a physical network interface 161 | 162 | If you have an extra physial network interface, you can test this application with it. 163 | 164 | **WARNING: you may lose the reachability to your machine if you make a wrong configuration. so, please conduct the following procedure only when you understand what you are doing.** 165 | 166 | The following command shows the currently availabe network interfaces. 167 | 168 | ``` 169 | ./dpdk/dpdk-22.03/usertools/dpdk-devbind.py -s 170 | ``` 171 | 172 | You will see like as follows. ( if you do not see this kind of output, it may mean that your machine does not have the interfaces that can be attached to DPDK. ) 173 | 174 | ``` 175 | Network devices using kernel driver 176 | =================================== 177 | 0000:04:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' if=ens3f0 drv=ixgbe unused=vfio-pci 178 | 0000:04:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' if=ens3f1 drv=ixgbe unused=vfio-pci 179 | ``` 180 | 181 | Let's say, we wish to use the 10 Gbps network interface identified by 0000:04:00.1. ( please select according to your environment. ) 182 | 183 | The output above says that 0000:04:00.1 is currently managed by the kernel-space ixgbe driver. 184 | 185 | To use it with DPDK, we need to bind it with the vfio-pci driver, and the following command does it. 186 | 187 | ``` 188 | sudo ./dpdk/dpdk-22.03/usertools/dpdk-devbind.py -b vfio-pci 0000:04:00.1 189 | ``` 190 | 191 | Afterward, you will see the following if you do ```dpdk-devbind.py -s``` above again; this means that 0000:04:00.1 is bound with vfio-pci. 192 | 193 | ``` 194 | Network devices using DPDK-compatible driver 195 | ============================================ 196 | 0000:04:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' drv=vfio-pci unused=ixgbe 197 | ``` 198 | 199 | Then, the following command launches the application using the 0000:04:00.1 network interface. 200 | 201 | ``` 202 | sudo LD_LIBRARY_PATH=./dpdk/install/lib/x86_64-linux-gnu ./app -l 0-1 --proc-type=primary --file-prefix=pmd1 --allow=0000:04:00.1 -- -a 10.0.0.2 -g 10.0.0.1 -m 255.255.255.0 -l 1 -p 10000 203 | ``` 204 | 205 | ## Rough numbers 206 | 207 | Here, we provide rough numbers and comparison with the default Linux TCP stack. 208 | 209 | ### Comparison with Linux 210 | 211 | For the Linux TCP stack test, we use the following C program, that is conceptionally equivalent to our lwIP and DPDK applied application. 212 | 213 | ``` 214 | #include 215 | #include 216 | #include 217 | #include 218 | #include 219 | #include 220 | #include 221 | 222 | #include 223 | #include 224 | #include 225 | 226 | #include 227 | #include 228 | #include 229 | 230 | int main(int argc, char* const* argv) 231 | { 232 | int ch; 233 | int port = 10000; 234 | size_t content_len = 1, httpdatalen; 235 | int epfd, sockfd; 236 | int on; 237 | char *httpbuf = NULL; 238 | 239 | while ((ch = getopt(argc, argv, "p:l:")) != -1) { 240 | switch (ch) { 241 | case 'l': 242 | content_len = atoi(optarg); 243 | break; 244 | case 'p': 245 | port = atoi(optarg); 246 | break; 247 | default: 248 | assert(0); 249 | break; 250 | } 251 | } 252 | 253 | { 254 | size_t buflen = content_len + 256 /* for http hdr */; 255 | char *content; 256 | assert((httpbuf = (char *) malloc(buflen)) != NULL); 257 | assert((content = (char *) malloc(content_len + 1)) != NULL); 258 | memset(content, 'A', content_len); 259 | content[content_len] = '\0'; 260 | httpdatalen = snprintf(httpbuf, buflen, "HTTP/1.1 200 OK\r\nContent-Length: %lu\r\nConnection: keep-alive\r\n\r\n%s", 261 | content_len, content); 262 | free(content); 263 | printf("http data length: %lu bytes\n", httpdatalen); 264 | } 265 | 266 | assert((epfd = epoll_create1(EPOLL_CLOEXEC)) != -1); 267 | assert((sockfd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) != -1); 268 | on = 1; 269 | assert(!setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on))); 270 | on = 1; 271 | assert(!setsockopt(sockfd, SOL_TCP, TCP_NODELAY, &on, sizeof(on))); 272 | on = 1; 273 | assert(!ioctl(sockfd, FIONBIO, &on)); 274 | { 275 | struct sockaddr_in sin = { 276 | .sin_family = AF_INET, 277 | .sin_addr.s_addr = htonl(INADDR_ANY), 278 | .sin_port = htons(port), 279 | }; 280 | assert(!bind(sockfd, (struct sockaddr *) &sin, sizeof(sin))); 281 | } 282 | assert(!listen(sockfd, SOMAXCONN)); 283 | { 284 | struct epoll_event ev = { 285 | .events = EPOLLIN, 286 | .data.fd = sockfd, 287 | }; 288 | assert(!epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev)); 289 | } 290 | 291 | while (1) { 292 | struct epoll_event evts[256]; 293 | int i, nfd = epoll_wait(epfd, evts, 256, -1); 294 | for (i = 0; i < nfd; i++) { 295 | if (evts[i].data.fd == sockfd) { 296 | struct sockaddr_in caddr_in; 297 | socklen_t addrlen; 298 | int newfd; 299 | while ((newfd = accept(evts[i].data.fd, (struct sockaddr *)&caddr_in, &addrlen)) != -1) { 300 | struct epoll_event ev = { 301 | .events = EPOLLIN, 302 | .data.fd = newfd, 303 | }; 304 | assert(!epoll_ctl(epfd, EPOLL_CTL_ADD, newfd, &ev)); 305 | } 306 | } else { 307 | ssize_t len; 308 | static char buf[70000]; 309 | len = read(evts[i].data.fd, buf, sizeof(buf)); 310 | if (len == 0) { 311 | close(evts[i].data.fd); 312 | continue; 313 | } else if (len < 0) 314 | continue; 315 | if (strncmp(buf, "GET ", strlen("GET ")) == 0) 316 | assert(httpdatalen == write(evts[i].data.fd, httpbuf, httpdatalen)); 317 | } 318 | } 319 | } 320 | 321 | free(httpbuf); 322 | 323 | return 0; 324 | } 325 | ``` 326 | 327 | ### Workload 328 | 329 | The server applications serve the static content, and the entier HTTP message, including the HTTP header, is 64 bytes. 330 | 331 | We change the number of concurrent client connections of wrk from 1 to 128. 332 | 333 | ``` 334 | $ ./wrk http://10.0.0.2:10000/ -d 10 -t 1 -c 1 -L 335 | $ ./wrk http://10.0.0.2:10000/ -d 10 -t 2 -c 2 -L 336 | $ ./wrk http://10.0.0.2:10000/ -d 10 -t 4 -c 4 -L 337 | $ ./wrk http://10.0.0.2:10000/ -d 10 -t 8 -c 8 -L 338 | $ ./wrk http://10.0.0.2:10000/ -d 10 -t 16 -c 16 -L 339 | $ ./wrk http://10.0.0.2:10000/ -d 10 -t 16 -c 32 -L 340 | $ ./wrk http://10.0.0.2:10000/ -d 10 -t 16 -c 64 -L 341 | $ ./wrk http://10.0.0.2:10000/ -d 10 -t 16 -c 128 -L 342 | ``` 343 | 344 | ### Hardware 345 | 346 | Using two machines, and each has: 347 | - CPU: 2 x 8-core Intel Xeon CPU E5-2640 v3 @ 2.60GHz 348 | - NIC: Intel 82599ES 10-Gigabit 349 | 350 | The two machines are directly connected via the 10 Gbps NICs. 351 | 352 | ### Result 353 | 354 | lwIP on DPDK achieves 51.7~483.5% higher throughput compared to the Linux TCP stack. 355 | 356 | Throughput ( Requests/sec ) 357 | 358 | | Concurrent connections | Linux TCP | lwIP on DPDK | 359 | | ------------- | ------------- | ------------- | 360 | | 1 | 19688.15 | 29871.01 | 361 | | 2 | 36760.34 | 60116.91 | 362 | | 4 | 39310.89 | 128728.74 | 363 | | 8 | 74619.08 | 253518.76 | 364 | | 16 | 123529.06 | 457784.18 | 365 | | 32 | 187544.78 | 795987.05 | 366 | | 64 | 189746.58 | 1107349.60 | 367 | | 128 | 189733.81 | 870747.60 | 368 | 369 | -------------------------------------------------------------------------------- /lwipopts.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2001-2003 Swedish Institute of Computer Science. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, 6 | * are permitted provided that the following conditions are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright notice, 11 | * this list of conditions and the following disclaimer in the documentation 12 | * and/or other materials provided with the distribution. 13 | * 3. The name of the author may not be used to endorse or promote products 14 | * derived from this software without specific prior written permission. 15 | * 16 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 19 | * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 20 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 21 | * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 24 | * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 25 | * OF SUCH DAMAGE. 26 | * 27 | * This file is part of the lwIP TCP/IP stack. 28 | * 29 | * Author: Adam Dunkels 30 | * 31 | */ 32 | #ifndef LWIP_LWIPOPTS_H 33 | #define LWIP_LWIPOPTS_H 34 | 35 | #ifdef LWIP_OPTTEST_FILE 36 | #include "lwipopts_test.h" 37 | #else /* LWIP_OPTTEST_FILE */ 38 | 39 | #define LWIP_IPV4 1 40 | #define LWIP_IPV6 0 41 | 42 | #define NO_SYS 1 43 | #define LWIP_SOCKET (NO_SYS==0) 44 | #define LWIP_NETCONN (NO_SYS==0) 45 | #define LWIP_NETIF_API (NO_SYS==0) 46 | 47 | #define LWIP_IGMP LWIP_IPV4 48 | #define LWIP_ICMP LWIP_IPV4 49 | 50 | #define LWIP_SNMP LWIP_UDP 51 | #define MIB2_STATS LWIP_SNMP 52 | #ifdef LWIP_HAVE_MBEDTLS 53 | #define LWIP_SNMP_V3 (LWIP_SNMP) 54 | #endif 55 | 56 | #define LWIP_DNS LWIP_UDP 57 | #define LWIP_MDNS_RESPONDER LWIP_UDP 58 | 59 | #define LWIP_NUM_NETIF_CLIENT_DATA (LWIP_MDNS_RESPONDER) 60 | 61 | #define LWIP_HAVE_LOOPIF 1 62 | #define LWIP_NETIF_LOOPBACK 1 63 | #define LWIP_LOOPBACK_MAX_PBUFS 10 64 | 65 | #define TCP_LISTEN_BACKLOG 1 66 | 67 | #define LWIP_COMPAT_SOCKETS 1 68 | #define LWIP_SO_RCVTIMEO 1 69 | #define LWIP_SO_RCVBUF 1 70 | 71 | #define LWIP_TCPIP_CORE_LOCKING 1 72 | 73 | #define LWIP_NETIF_LINK_CALLBACK 1 74 | #define LWIP_NETIF_STATUS_CALLBACK 1 75 | #define LWIP_NETIF_EXT_STATUS_CALLBACK 1 76 | 77 | #define LWIP_DEBUG 1 78 | #ifdef LWIP_DEBUG 79 | 80 | #define LWIP_DBG_MIN_LEVEL 0 81 | #define PPP_DEBUG LWIP_DBG_OFF 82 | #define MEM_DEBUG LWIP_DBG_OFF 83 | #define MEMP_DEBUG LWIP_DBG_OFF 84 | #define PBUF_DEBUG LWIP_DBG_OFF 85 | #define API_LIB_DEBUG LWIP_DBG_OFF 86 | #define API_MSG_DEBUG LWIP_DBG_OFF 87 | #define TCPIP_DEBUG LWIP_DBG_OFF 88 | #define NETIF_DEBUG LWIP_DBG_OFF 89 | #define SOCKETS_DEBUG LWIP_DBG_OFF 90 | #define DNS_DEBUG LWIP_DBG_OFF 91 | #define AUTOIP_DEBUG LWIP_DBG_OFF 92 | #define DHCP_DEBUG LWIP_DBG_OFF 93 | #define IP_DEBUG LWIP_DBG_OFF 94 | #define IP_REASS_DEBUG LWIP_DBG_OFF 95 | #define ICMP_DEBUG LWIP_DBG_OFF 96 | #define IGMP_DEBUG LWIP_DBG_OFF 97 | #define UDP_DEBUG LWIP_DBG_OFF 98 | #define TCP_DEBUG LWIP_DBG_OFF 99 | #define TCP_INPUT_DEBUG LWIP_DBG_OFF 100 | #define TCP_OUTPUT_DEBUG LWIP_DBG_OFF 101 | #define TCP_RTO_DEBUG LWIP_DBG_OFF 102 | #define TCP_CWND_DEBUG LWIP_DBG_OFF 103 | #define TCP_WND_DEBUG LWIP_DBG_OFF 104 | #define TCP_FR_DEBUG LWIP_DBG_OFF 105 | #define TCP_QLEN_DEBUG LWIP_DBG_OFF 106 | #define TCP_RST_DEBUG LWIP_DBG_OFF 107 | #endif 108 | 109 | #define LWIP_DBG_TYPES_ON (LWIP_DBG_ON|LWIP_DBG_TRACE|LWIP_DBG_STATE|LWIP_DBG_FRESH|LWIP_DBG_HALT) 110 | 111 | 112 | /* ---------- Memory options ---------- */ 113 | /* MEM_ALIGNMENT: should be set to the alignment of the CPU for which 114 | lwIP is compiled. 4 byte alignment -> define MEM_ALIGNMENT to 4, 2 115 | byte alignment -> define MEM_ALIGNMENT to 2. */ 116 | /* MSVC port: intel processors don't need 4-byte alignment, 117 | but are faster that way! */ 118 | #define MEM_ALIGNMENT 4U 119 | 120 | /* MEM_SIZE: the size of the heap memory. If the application will send 121 | a lot of data that needs to be copied, this should be set high. */ 122 | #define MEM_SIZE (1L<<30) 123 | 124 | /* MEMP_NUM_PBUF: the number of memp struct pbufs. If the application 125 | sends a lot of data out of ROM (or other static memory), this 126 | should be set high. */ 127 | #define MEMP_NUM_PBUF (32768) 128 | /* MEMP_NUM_RAW_PCB: the number of UDP protocol control blocks. One 129 | per active RAW "connection". */ 130 | #define MEMP_NUM_RAW_PCB 128 131 | /* MEMP_NUM_UDP_PCB: the number of UDP protocol control blocks. One 132 | per active UDP "connection". */ 133 | #define MEMP_NUM_UDP_PCB 128 134 | /* MEMP_NUM_TCP_PCB: the number of simulatenously active TCP 135 | connections. */ 136 | #define MEMP_NUM_TCP_PCB 256 137 | /* MEMP_NUM_TCP_PCB_LISTEN: the number of listening TCP 138 | connections. */ 139 | #define MEMP_NUM_TCP_PCB_LISTEN 32 140 | /* MEMP_NUM_TCP_SEG: the number of simultaneously queued TCP 141 | segments. */ 142 | #define MEMP_NUM_TCP_SEG 32768 143 | /* MEMP_NUM_SYS_TIMEOUT: the number of simulateously active 144 | timeouts. */ 145 | #define MEMP_NUM_SYS_TIMEOUT 17 146 | 147 | /* The following four are used only with the sequential API and can be 148 | set to 0 if the application only will use the raw API. */ 149 | /* MEMP_NUM_NETBUF: the number of struct netbufs. */ 150 | #define MEMP_NUM_NETBUF 8192 151 | /* MEMP_NUM_NETCONN: the number of struct netconns. */ 152 | #define MEMP_NUM_NETCONN 512 153 | /* MEMP_NUM_TCPIP_MSG_*: the number of struct tcpip_msg, which is used 154 | for sequential API communication and incoming packets. Used in 155 | src/api/tcpip.c. */ 156 | #define MEMP_NUM_TCPIP_MSG_API 512 157 | #define MEMP_NUM_TCPIP_MSG_INPKT 512 158 | 159 | 160 | /* ---------- Pbuf options ---------- */ 161 | /* PBUF_POOL_SIZE: the number of buffers in the pbuf pool. */ 162 | #define PBUF_POOL_SIZE 8192 163 | 164 | /* PBUF_POOL_BUFSIZE: the size of each pbuf in the pbuf pool. */ 165 | #define PBUF_POOL_BUFSIZE 8192 166 | 167 | /** SYS_LIGHTWEIGHT_PROT 168 | * define SYS_LIGHTWEIGHT_PROT in lwipopts.h if you want inter-task protection 169 | * for certain critical regions during buffer allocation, deallocation and memory 170 | * allocation and deallocation. 171 | */ 172 | #define SYS_LIGHTWEIGHT_PROT (NO_SYS==0) 173 | 174 | 175 | /* ---------- TCP options ---------- */ 176 | #define LWIP_TCP 1 177 | #define TCP_TTL 255 178 | 179 | #define LWIP_ALTCP (LWIP_TCP) 180 | #ifdef LWIP_HAVE_MBEDTLS 181 | #define LWIP_ALTCP_TLS (LWIP_TCP) 182 | #define LWIP_ALTCP_TLS_MBEDTLS (LWIP_TCP) 183 | #endif 184 | 185 | 186 | /* Controls if TCP should queue segments that arrive out of 187 | order. Define to 0 if your device is low on memory. */ 188 | #define TCP_QUEUE_OOSEQ 1 189 | 190 | /* TCP Maximum segment size. */ 191 | #define TCP_MSS 1460 192 | 193 | /* TCP sender buffer space (bytes). */ 194 | #define TCP_SND_BUF (65535) 195 | 196 | /* TCP sender buffer space (pbufs). This must be at least = 2 * 197 | TCP_SND_BUF/TCP_MSS for things to work. */ 198 | #define TCP_SND_QUEUELEN (2 * TCP_SND_BUF/TCP_MSS) 199 | 200 | /* TCP writable space (bytes). This must be less than or equal 201 | to TCP_SND_BUF. It is the amount of space which must be 202 | available in the tcp snd_buf for select to return writable */ 203 | #define TCP_SNDLOWAT (TCP_SND_BUF/2) 204 | 205 | /* TCP receive window. */ 206 | #define TCP_WND (20 * 1024) 207 | 208 | /* Maximum number of retransmissions of data segments. */ 209 | #define TCP_MAXRTX 12 210 | 211 | /* Maximum number of retransmissions of SYN segments. */ 212 | #define TCP_SYNMAXRTX 4 213 | 214 | 215 | /* ---------- ARP options ---------- */ 216 | #define LWIP_ARP 1 217 | #define ARP_TABLE_SIZE 10 218 | #define ARP_QUEUEING 1 219 | 220 | 221 | /* ---------- IP options ---------- */ 222 | /* Define IP_FORWARD to 1 if you wish to have the ability to forward 223 | IP packets across network interfaces. If you are going to run lwIP 224 | on a device with only one network interface, define this to 0. */ 225 | #define IP_FORWARD 1 226 | 227 | /* IP reassembly and segmentation.These are orthogonal even 228 | * if they both deal with IP fragments */ 229 | #define IP_REASSEMBLY 1 230 | #define IP_REASS_MAX_PBUFS (10 * ((1500 + PBUF_POOL_BUFSIZE - 1) / PBUF_POOL_BUFSIZE)) 231 | #define MEMP_NUM_REASSDATA IP_REASS_MAX_PBUFS 232 | #define IP_FRAG 1 233 | #define IPV6_FRAG_COPYHEADER 1 234 | 235 | /* ---------- ICMP options ---------- */ 236 | #define ICMP_TTL 255 237 | 238 | 239 | /* ---------- DHCP options ---------- */ 240 | /* Define LWIP_DHCP to 1 if you want DHCP configuration of 241 | interfaces. */ 242 | #define LWIP_DHCP LWIP_UDP 243 | 244 | /* 1 if you want to do an ARP check on the offered address 245 | (recommended). */ 246 | #define DHCP_DOES_ARP_CHECK (LWIP_DHCP) 247 | 248 | 249 | /* ---------- AUTOIP options ------- */ 250 | #define LWIP_AUTOIP (LWIP_DHCP) 251 | #define LWIP_DHCP_AUTOIP_COOP (LWIP_DHCP && LWIP_AUTOIP) 252 | 253 | 254 | /* ---------- UDP options ---------- */ 255 | #define LWIP_UDP 1 256 | #define LWIP_UDPLITE LWIP_UDP 257 | #define UDP_TTL 255 258 | 259 | 260 | /* ---------- RAW options ---------- */ 261 | #define LWIP_RAW 1 262 | 263 | 264 | /* ---------- Statistics options ---------- */ 265 | 266 | #define LWIP_STATS 1 267 | #define LWIP_STATS_DISPLAY 1 268 | 269 | #if LWIP_STATS 270 | #define LINK_STATS 1 271 | #define IP_STATS 1 272 | #define ICMP_STATS 1 273 | #define IGMP_STATS 1 274 | #define IPFRAG_STATS 1 275 | #define UDP_STATS 1 276 | #define TCP_STATS 1 277 | #define MEM_STATS 1 278 | #define MEMP_STATS 1 279 | #define PBUF_STATS 1 280 | #define SYS_STATS 1 281 | #endif /* LWIP_STATS */ 282 | 283 | /* ---------- NETBIOS options ---------- */ 284 | #define LWIP_NETBIOS_RESPOND_NAME_QUERY 1 285 | 286 | /* ---------- PPP options ---------- */ 287 | 288 | #define PPP_SUPPORT 0 /* Set > 0 for PPP */ 289 | 290 | #if PPP_SUPPORT 291 | 292 | #define NUM_PPP 1 /* Max PPP sessions. */ 293 | 294 | 295 | /* Select modules to enable. Ideally these would be set in the makefile but 296 | * we're limited by the command line length so you need to modify the settings 297 | * in this file. 298 | */ 299 | #define PPPOE_SUPPORT 1 300 | #define PPPOS_SUPPORT 1 301 | 302 | #define PAP_SUPPORT 1 /* Set > 0 for PAP. */ 303 | #define CHAP_SUPPORT 1 /* Set > 0 for CHAP. */ 304 | #define MSCHAP_SUPPORT 0 /* Set > 0 for MSCHAP */ 305 | #define CBCP_SUPPORT 0 /* Set > 0 for CBCP (NOT FUNCTIONAL!) */ 306 | #define CCP_SUPPORT 0 /* Set > 0 for CCP */ 307 | #define VJ_SUPPORT 1 /* Set > 0 for VJ header compression. */ 308 | #define MD5_SUPPORT 1 /* Set > 0 for MD5 (see also CHAP) */ 309 | 310 | #endif /* PPP_SUPPORT */ 311 | 312 | #endif /* LWIP_OPTTEST_FILE */ 313 | 314 | /* The following defines must be done even in OPTTEST mode: */ 315 | 316 | #if !defined(NO_SYS) || !NO_SYS /* default is 0 */ 317 | void sys_check_core_locking(void); 318 | #define LWIP_ASSERT_CORE_LOCKED() sys_check_core_locking() 319 | void sys_mark_tcpip_thread(void); 320 | #define LWIP_MARK_TCPIP_THREAD() sys_mark_tcpip_thread() 321 | 322 | #if !defined(LWIP_TCPIP_CORE_LOCKING) || LWIP_TCPIP_CORE_LOCKING /* default is 1 */ 323 | void sys_lock_tcpip_core(void); 324 | #define LOCK_TCPIP_CORE() sys_lock_tcpip_core() 325 | void sys_unlock_tcpip_core(void); 326 | #define UNLOCK_TCPIP_CORE() sys_unlock_tcpip_core() 327 | #endif 328 | #endif 329 | 330 | 331 | /* added for the benchmark app */ 332 | 333 | #define LWIP_DONT_PROVIDE_BYTEORDER_FUNCTIONS 1 334 | 335 | #define LWIP_TCP_KEEPALIVE 1 336 | 337 | #define LWIP_TCP_PCB_NUM_EXT_ARGS 1 338 | 339 | #endif /* LWIP_LWIPOPTS_H */ 340 | -------------------------------------------------------------------------------- /main.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2022 Kenichi Yasukata 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | 47 | /* workaround to avoid conflicts between dpdk and lwip definitions */ 48 | #undef IP_DF 49 | #undef IP_MF 50 | #undef IP_RF 51 | #undef IP_OFFMASK 52 | 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | #include 60 | #include 61 | #include 62 | 63 | #include 64 | 65 | #define MAX_PKT_BURST (32) 66 | #define NUM_SLOT (256) 67 | 68 | #define MEMPOOL_CACHE_SIZE (256) 69 | 70 | #define PACKET_BUF_SIZE (1518) 71 | 72 | static struct rte_mempool *pktmbuf_pool = NULL; 73 | static int tx_idx = 0; 74 | static struct rte_mbuf *tx_mbufs[MAX_PKT_BURST] = { 0 }; 75 | 76 | static char *httpbuf; 77 | static size_t httpdatalen; 78 | 79 | static void tx_flush(void) 80 | { 81 | int xmit = tx_idx, xmitted = 0; 82 | while (xmitted != xmit) 83 | xmitted += rte_eth_tx_burst(0 /* port id */, 0 /* queue id */, &tx_mbufs[xmitted], xmit - xmitted); 84 | tx_idx = 0; 85 | } 86 | 87 | static err_t low_level_output(struct netif *netif __attribute__((unused)), struct pbuf *p) 88 | { 89 | char buf[PACKET_BUF_SIZE]; 90 | void *bufptr, *largebuf = NULL; 91 | if (sizeof(buf) < p->tot_len) { 92 | largebuf = (char *) malloc(p->tot_len); 93 | assert(largebuf); 94 | bufptr = largebuf; 95 | } else 96 | bufptr = buf; 97 | 98 | pbuf_copy_partial(p, bufptr, p->tot_len, 0); 99 | 100 | assert((tx_mbufs[tx_idx] = rte_pktmbuf_alloc(pktmbuf_pool)) != NULL); 101 | assert(p->tot_len <= RTE_MBUF_DEFAULT_BUF_SIZE); 102 | rte_memcpy(rte_pktmbuf_mtod(tx_mbufs[tx_idx], void *), bufptr, p->tot_len); 103 | rte_pktmbuf_pkt_len(tx_mbufs[tx_idx]) = rte_pktmbuf_data_len(tx_mbufs[tx_idx]) = p->tot_len; 104 | if (++tx_idx == MAX_PKT_BURST) 105 | tx_flush(); 106 | 107 | if (largebuf) 108 | free(largebuf); 109 | return ERR_OK; 110 | } 111 | 112 | static unsigned long io_stat[3] = { 0 }; 113 | 114 | struct http_response { 115 | int state; 116 | long content_tot_len; 117 | long content_recvd; 118 | long cur; 119 | char buf[1UL << 16]; 120 | }; 121 | 122 | static err_t tcp_recv_handler(void *arg, struct tcp_pcb *tpcb, 123 | struct pbuf *p, err_t err) 124 | { 125 | if (err != ERR_OK) 126 | return err; 127 | if (!p) { 128 | tcp_close(tpcb); 129 | return ERR_OK; 130 | } 131 | io_stat[1] += p->tot_len; 132 | if (!arg) { /* server mode */ 133 | char buf[4] = { 0 }; 134 | pbuf_copy_partial(p, buf, 3, 0); 135 | if (!strncmp(buf, "GET", 3)) { 136 | io_stat[0]++; 137 | io_stat[2] += httpdatalen; 138 | assert(tcp_sndbuf(tpcb) >= httpdatalen); 139 | assert(tcp_write(tpcb, httpbuf, httpdatalen, TCP_WRITE_FLAG_COPY) == ERR_OK); 140 | assert(tcp_output(tpcb) == ERR_OK); 141 | } 142 | } else { /* client mode */ 143 | struct http_response *r = (struct http_response *) arg; 144 | assert(p->tot_len < (sizeof(r->buf) - r->cur)); 145 | pbuf_copy_partial(p, &r->buf[r->cur], p->tot_len, 0); 146 | r->cur += p->tot_len; 147 | switch (r->state) { 148 | case 0: 149 | { 150 | long i; 151 | for (i = 0; i < r->cur && r->state == 0; i++) { 152 | if (r->buf[i] == 'C') { 153 | if (r->cur - i > 15) { 154 | if (!memcmp(&r->buf[i], "Content-Length:", 15)) { 155 | long j; 156 | for (j = 0; j < (r->cur - i - 15); j++) { 157 | if (r->buf[i + 15 + j] == '\r') { 158 | r->buf[i + 15 + j] = '\0'; 159 | assert(sscanf(&r->buf[i], "Content-Length: %ld", &r->content_tot_len) == 1); 160 | r->buf[i + 15 + j] = '\r'; 161 | r->state = 1; 162 | break; 163 | } 164 | } 165 | } 166 | } 167 | } 168 | } 169 | } 170 | /* fall through */ 171 | case 1: 172 | { 173 | long i; 174 | for (i = 0; i <= r->cur - 4 && r->state == 1; i++) { 175 | if (r->buf[i + 0] == '\r' && r->buf[i + 1] == '\n' && r->buf[i + 2] == '\r' && r->buf[i + 3] == '\n') { 176 | r->cur -= i + 4; 177 | r->content_recvd = 0; 178 | r->state = 2; 179 | break; 180 | } 181 | } 182 | } 183 | /* fall through */ 184 | case 2: 185 | r->content_recvd += r->cur; 186 | if (r->content_recvd == r->content_tot_len) { 187 | io_stat[0]++; 188 | io_stat[2] += 42; 189 | assert(tcp_sndbuf(tpcb) >= 42); 190 | assert(tcp_write(tpcb, "GET / HTTP/1.0\r\nConnection: Keep-Alive\r\n\r\n", 42, TCP_WRITE_FLAG_COPY) == ERR_OK); 191 | assert(tcp_output(tpcb) == ERR_OK); 192 | r->state = 0; 193 | } 194 | r->cur = 0; 195 | break; 196 | default: 197 | assert(0); 198 | break; 199 | } 200 | } 201 | tcp_recved(tpcb, p->tot_len); 202 | pbuf_free(p); 203 | return ERR_OK; 204 | } 205 | 206 | static void tcp_destroy_handeler(unsigned char id __attribute__((unused)), void *data) 207 | { 208 | if (data) 209 | free(data); 210 | } 211 | 212 | static const struct tcp_ext_arg_callbacks tcp_ext_arg_cbs = { 213 | .destroy = tcp_destroy_handeler, 214 | }; 215 | 216 | static err_t accept_handler(void *arg __attribute__((unused)), struct tcp_pcb *tpcb, err_t err) 217 | { 218 | if (err != ERR_OK) 219 | return err; 220 | 221 | tcp_recv(tpcb, tcp_recv_handler); 222 | tcp_setprio(tpcb, TCP_PRIO_MAX); 223 | 224 | tcp_ext_arg_set_callbacks(tpcb, 0, &tcp_ext_arg_cbs); 225 | tcp_ext_arg_set(tpcb, 0, NULL); 226 | 227 | tcp_nagle_disable(tpcb); 228 | 229 | tpcb->so_options |= SOF_KEEPALIVE; 230 | tpcb->keep_intvl = (60 * 1000); 231 | tpcb->keep_idle = (60 * 1000); 232 | tpcb->keep_cnt = 1; 233 | 234 | return err; 235 | } 236 | 237 | static err_t connected_handler(void *arg, struct tcp_pcb *tpcb, err_t err) 238 | { 239 | if (err != ERR_OK) 240 | return err; 241 | if ((err = accept_handler(arg, tpcb, err)) != ERR_OK) 242 | return err; 243 | 244 | io_stat[2] += 42; 245 | assert(tcp_sndbuf(tpcb) >= 42); 246 | assert(tcp_write(tpcb, "GET / HTTP/1.0\r\nConnection: Keep-Alive\r\n\r\n", 42, TCP_WRITE_FLAG_COPY) == ERR_OK); 247 | assert(tcp_output(tpcb) == ERR_OK); 248 | 249 | return ERR_OK; 250 | } 251 | 252 | static err_t if_init(struct netif *netif) 253 | { 254 | { 255 | struct rte_ether_addr ports_eth_addr; 256 | assert(rte_eth_macaddr_get(0 /* port id */, &ports_eth_addr) >= 0); 257 | for (int i = 0; i < 6; i++) 258 | netif->hwaddr[i] = ports_eth_addr.addr_bytes[i]; 259 | } 260 | { 261 | uint16_t _mtu; 262 | assert(rte_eth_dev_get_mtu(0 /* port id */, &_mtu) >= 0); 263 | assert(_mtu <= PACKET_BUF_SIZE); 264 | netif->mtu = _mtu; 265 | } 266 | netif->output = etharp_output; 267 | netif->linkoutput = low_level_output; 268 | netif->hwaddr_len = 6; 269 | netif->flags = NETIF_FLAG_BROADCAST | NETIF_FLAG_ETHARP | NETIF_FLAG_IGMP; 270 | return ERR_OK; 271 | } 272 | 273 | int main(int argc, char *const *argv) 274 | { 275 | struct netif _netif = { 0 }; 276 | ip4_addr_t _addr, _mask, _gate, _srv_ip; 277 | size_t content_len = 1; 278 | int server_port = 10000, num_conn = 1; 279 | bool mode_server = true; 280 | int max_epoll_wait_timeout_ms = 0; 281 | 282 | { 283 | int ret; 284 | assert((ret = rte_eal_init(argc, (char **) argv)) >= 0); 285 | argc -= ret; 286 | argv += ret; 287 | } 288 | 289 | assert(rte_eth_dev_count_avail() == 1); 290 | 291 | { 292 | int ch; 293 | bool _a = false, _g = false, _m = false; 294 | while ((ch = getopt(argc, argv, "a:c:e:g:l:m:p:s:")) != -1) { 295 | switch (ch) { 296 | case 'a': 297 | inet_pton(AF_INET, optarg, &_addr); 298 | _a = true; 299 | break; 300 | case 'c': 301 | num_conn = atoi(optarg); 302 | break; 303 | case 'e': 304 | assert(sscanf(optarg, "%d", &max_epoll_wait_timeout_ms) == 1); 305 | break; 306 | case 'g': 307 | inet_pton(AF_INET, optarg, &_gate); 308 | _g = true; 309 | break; 310 | case 'm': 311 | inet_pton(AF_INET, optarg, &_mask); 312 | _m = true; 313 | break; 314 | case 'l': 315 | content_len = atol(optarg); 316 | break; 317 | case 'p': 318 | server_port = atoi(optarg); 319 | break; 320 | case 's': 321 | inet_pton(AF_INET, optarg, &_srv_ip); 322 | mode_server = false; 323 | break; 324 | default: 325 | assert(0); 326 | break; 327 | } 328 | } 329 | assert(_a && _g && _m); 330 | } 331 | 332 | { 333 | uint16_t nb_rxd = NUM_SLOT; 334 | uint16_t nb_txd = NUM_SLOT; 335 | assert((pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", 336 | RTE_MAX(1 /* nb_ports */ * (nb_rxd + nb_txd + MAX_PKT_BURST + 1 * MEMPOOL_CACHE_SIZE), 8192), 337 | MEMPOOL_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, 338 | rte_socket_id())) != NULL); 339 | 340 | { 341 | struct rte_eth_dev_info dev_info; 342 | struct rte_eth_conf local_port_conf = { 0 }; 343 | 344 | assert(rte_eth_dev_info_get(0 /* port id */, &dev_info) >= 0); 345 | 346 | if (max_epoll_wait_timeout_ms) 347 | local_port_conf.intr_conf.rxq = 1; 348 | 349 | assert(rte_eth_dev_configure(0 /* port id */, 1 /* num queues */, 1 /* num queues */, &local_port_conf) >= 0); 350 | 351 | assert(rte_eth_dev_adjust_nb_rx_tx_desc(0 /* port id */, &nb_rxd, &nb_txd) >= 0); 352 | 353 | assert(rte_eth_rx_queue_setup(0 /* port id */, 0 /* queue */, nb_rxd, 354 | rte_eth_dev_socket_id(0 /* port id */), 355 | &dev_info.default_rxconf, 356 | pktmbuf_pool) >= 0); 357 | 358 | assert(rte_eth_tx_queue_setup(0 /* port id */, 0 /* queue */, nb_txd, 359 | rte_eth_dev_socket_id(0 /* port id */), 360 | &dev_info.default_txconf) >= 0); 361 | 362 | assert(rte_eth_dev_start(0 /* port id */) >= 0); 363 | assert(rte_eth_promiscuous_enable(0 /* port id */) >= 0); 364 | 365 | if (max_epoll_wait_timeout_ms) 366 | assert(!rte_eth_dev_rx_intr_ctl_q(0 /* port id */, 0 /* queue */, RTE_EPOLL_PER_THREAD, RTE_INTR_EVENT_ADD, NULL)); 367 | } 368 | } 369 | 370 | /* setting up lwip */ 371 | { 372 | lwip_init(); 373 | assert(netif_add(&_netif, &_addr, &_mask, &_gate, NULL, if_init, ethernet_input) != NULL); 374 | netif_set_default(&_netif); 375 | netif_set_link_up(&_netif); 376 | netif_set_up(&_netif); 377 | } 378 | 379 | if (mode_server) { /* server mode */ 380 | { 381 | size_t buflen = content_len + 256 /* for http hdr */; 382 | char *content; 383 | assert((httpbuf = (char *) malloc(buflen)) != NULL); 384 | assert((content = (char *) malloc(content_len + 1)) != NULL); 385 | memset(content, 'A', content_len); 386 | content[content_len] = '\0'; 387 | httpdatalen = snprintf(httpbuf, buflen, "HTTP/1.1 200 OK\r\nContent-Length: %lu\r\nConnection: keep-alive\r\n\r\n%s", content_len, content); 388 | free(content); 389 | printf("http data length: %lu bytes\n", httpdatalen); 390 | } 391 | { 392 | struct tcp_pcb *tpcb, *_tpcb; 393 | assert((_tpcb = tcp_new()) != NULL); 394 | assert(tcp_bind(_tpcb, IP_ADDR_ANY, server_port) == ERR_OK); 395 | assert((tpcb = tcp_listen(_tpcb)) != NULL); 396 | tcp_accept(tpcb, accept_handler); 397 | tcp_ext_arg_set_callbacks(tpcb, 0, &tcp_ext_arg_cbs); 398 | tcp_ext_arg_set(tpcb, 0, NULL); 399 | } 400 | } else { /* client mode */ 401 | int i; 402 | printf("%d concurrent connection(s)\n", num_conn); 403 | for (i = 0; i < num_conn; i++) { 404 | struct tcp_pcb *tpcb; 405 | assert((tpcb = tcp_new()) != NULL); 406 | { 407 | struct http_response *r; 408 | assert((r = (struct http_response *) malloc(sizeof(struct http_response))) != NULL); 409 | r->state = 0; 410 | r->cur = 0; 411 | tcp_arg(tpcb, (void *) r); 412 | tcp_ext_arg_set(tpcb, 0, (void *) r); 413 | } 414 | assert(tcp_connect(tpcb, &_srv_ip, server_port, connected_handler) == ERR_OK); 415 | } 416 | } 417 | 418 | printf("-- application has started --\n"); 419 | 420 | /* primary loop */ 421 | { 422 | unsigned long prev_ts = 0; 423 | while (1) { 424 | struct rte_mbuf *rx_mbufs[MAX_PKT_BURST]; 425 | unsigned short i, nb_rx = rte_eth_rx_burst(0 /* port id */, 0 /* queue id */, rx_mbufs, MAX_PKT_BURST); 426 | for (i = 0; i < nb_rx; i++) { 427 | { 428 | struct pbuf *p; 429 | assert((p = pbuf_alloc(PBUF_RAW, rte_pktmbuf_pkt_len(rx_mbufs[i]), PBUF_POOL)) != NULL); 430 | pbuf_take(p, rte_pktmbuf_mtod(rx_mbufs[i], void *), rte_pktmbuf_pkt_len(rx_mbufs[i])); 431 | p->len = p->tot_len = rte_pktmbuf_pkt_len(rx_mbufs[i]); 432 | assert(_netif.input(p, &_netif) == ERR_OK); 433 | } 434 | rte_pktmbuf_free(rx_mbufs[i]); 435 | } 436 | tx_flush(); 437 | sys_check_timeouts(); 438 | { 439 | unsigned long now = ({ struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); (ts.tv_sec * 1000000000UL + ts.tv_nsec); }); 440 | if (now - prev_ts > 1000000000UL) { 441 | printf("[%s]: %10lu Requests/sec (rx %11lu bps, tx %11lu bbs)\n", 442 | (mode_server ? "server" : "client"), io_stat[0], io_stat[1] * 8, io_stat[2] * 8); 443 | memset(io_stat, 0, sizeof(io_stat)); 444 | prev_ts = now; 445 | } 446 | 447 | } 448 | if (!nb_rx && max_epoll_wait_timeout_ms) { 449 | assert(!rte_eth_dev_rx_intr_enable(0 /* port id */, 0 /* queue id */)); 450 | { 451 | struct rte_epoll_event ev; 452 | (void) rte_epoll_wait(RTE_EPOLL_PER_THREAD, &ev, 1, max_epoll_wait_timeout_ms < 0 ? 100 : (max_epoll_wait_timeout_ms > 100 ? 100 : max_epoll_wait_timeout_ms)); 453 | } 454 | rte_eth_dev_rx_intr_disable(0 /* port id */, 0 /* queue id */); 455 | } 456 | } 457 | } 458 | 459 | return 0; 460 | } 461 | --------------------------------------------------------------------------------