├── 02_send_recv ├── test │ ├── build │ │ ├── run │ │ └── run-shared │ ├── Makefile │ └── main.c ├── traffic_gen │ ├── build │ │ ├── run │ │ └── run-shared │ ├── recv_pkt.h │ ├── send_pkt.h │ ├── portinit.h │ ├── recv_pkt.c │ ├── common.h │ ├── Makefile │ ├── main.c │ ├── send_pkt.c │ └── portinit.c └── README.md ├── 03_ring ├── 02_pktRing │ ├── build │ │ ├── run │ │ └── run-shared │ ├── recv_pkt.h │ ├── send_pkt.h │ ├── common.h │ ├── recv_pkt.c │ ├── main.c │ ├── Makefile │ └── send_pkt.c ├── 01_simpleRing │ ├── build │ │ ├── run │ │ └── run-shared │ ├── Makefile │ └── main.c ├── 03_singleThreadPktRing │ ├── build │ │ ├── run │ │ └── run-shared │ ├── recv_pkt.h │ ├── send_pkt.h │ ├── portinit.h │ ├── ring.h │ ├── common.h │ ├── recv_pkt.c │ ├── ring.c │ ├── Makefile │ ├── main.c │ ├── send_pkt.c │ └── portinit.c ├── images │ ├── 1.png │ ├── 2.png │ ├── 3.png │ ├── 4.png │ ├── 5.png │ ├── 6.png │ ├── 7.png │ ├── 8.png │ └── 9.png ├── README2.md └── README.md ├── 04_rss ├── multiThreadRing │ ├── build │ │ ├── run │ │ └── run-shared │ ├── recv_pkt.h │ ├── send_pkt.h │ ├── portinit.h │ ├── ring.h │ ├── common.h │ ├── recv_pkt.c │ ├── ring.c │ ├── Makefile │ ├── main.c │ ├── send_pkt.c │ └── portinit.c └── README.md ├── README.md ├── 01_port_init ├── devinfo │ ├── build │ │ ├── devinfo │ │ └── devinfo-shared │ ├── Makefile │ └── main.c ├── test │ ├── build │ │ ├── portinit │ │ ├── helloworld │ │ └── portinit-shared │ ├── main.c │ └── Makefile └── README.md ├── a1_setup_mlx5_sriov_env ├── test │ ├── build │ │ ├── helloworld │ │ └── helloworld-shared │ ├── Makefile │ ├── main.c │ └── main.c.bak ├── sriov.md ├── ubuntu.md └── centos.md ├── a2_kvm ├── images │ ├── 1.png │ └── 2.png └── README.md ├── fix_format.sh ├── .vscode ├── c_cpp_properties.json └── settings.json ├── .gitignore ├── a3_crypto_qat └── README.md ├── LICENSE └── a4_octeon └── sdk_build.md /02_send_recv/test/build/run: -------------------------------------------------------------------------------- 1 | run-shared -------------------------------------------------------------------------------- /03_ring/02_pktRing/build/run: -------------------------------------------------------------------------------- 1 | run-shared -------------------------------------------------------------------------------- /03_ring/01_simpleRing/build/run: -------------------------------------------------------------------------------- 1 | run-shared -------------------------------------------------------------------------------- /04_rss/multiThreadRing/build/run: -------------------------------------------------------------------------------- 1 | run-shared -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # learn_dpdk 2 | Learn dpdk 3 | -------------------------------------------------------------------------------- /01_port_init/devinfo/build/devinfo: -------------------------------------------------------------------------------- 1 | devinfo-shared -------------------------------------------------------------------------------- /01_port_init/test/build/portinit: -------------------------------------------------------------------------------- 1 | portinit-shared -------------------------------------------------------------------------------- /02_send_recv/traffic_gen/build/run: -------------------------------------------------------------------------------- 1 | run-shared -------------------------------------------------------------------------------- /01_port_init/test/build/helloworld: -------------------------------------------------------------------------------- 1 | helloworld-shared -------------------------------------------------------------------------------- /03_ring/03_singleThreadPktRing/build/run: -------------------------------------------------------------------------------- 1 | run-shared -------------------------------------------------------------------------------- /a1_setup_mlx5_sriov_env/test/build/helloworld: -------------------------------------------------------------------------------- 1 | helloworld-shared -------------------------------------------------------------------------------- /a2_kvm/images/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/a2_kvm/images/1.png -------------------------------------------------------------------------------- /a2_kvm/images/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/a2_kvm/images/2.png -------------------------------------------------------------------------------- /03_ring/images/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/03_ring/images/1.png -------------------------------------------------------------------------------- /03_ring/images/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/03_ring/images/2.png -------------------------------------------------------------------------------- /03_ring/images/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/03_ring/images/3.png -------------------------------------------------------------------------------- /03_ring/images/4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/03_ring/images/4.png -------------------------------------------------------------------------------- /03_ring/images/5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/03_ring/images/5.png -------------------------------------------------------------------------------- /03_ring/images/6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/03_ring/images/6.png -------------------------------------------------------------------------------- /03_ring/images/7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/03_ring/images/7.png -------------------------------------------------------------------------------- /03_ring/images/8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/03_ring/images/8.png -------------------------------------------------------------------------------- /03_ring/images/9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/03_ring/images/9.png -------------------------------------------------------------------------------- /03_ring/02_pktRing/recv_pkt.h: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int lcore_recv_pkt(struct rte_ring *rx_ring); 4 | -------------------------------------------------------------------------------- /03_ring/02_pktRing/send_pkt.h: -------------------------------------------------------------------------------- 1 | 2 | #include "common.h" 3 | 4 | int lcore_send_pkt(struct lcore_params *p); -------------------------------------------------------------------------------- /04_rss/multiThreadRing/recv_pkt.h: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int lcore_recv_pkt(struct rx_params *rx); 4 | -------------------------------------------------------------------------------- /02_send_recv/traffic_gen/recv_pkt.h: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int lcore_recv_pkt(struct lcore_params *rx); 4 | -------------------------------------------------------------------------------- /02_send_recv/traffic_gen/send_pkt.h: -------------------------------------------------------------------------------- 1 | 2 | #include "common.h" 3 | 4 | int lcore_send_pkt(struct lcore_params *p); -------------------------------------------------------------------------------- /04_rss/multiThreadRing/send_pkt.h: -------------------------------------------------------------------------------- 1 | 2 | #include "common.h" 3 | 4 | int lcore_send_pkt(struct lcore_params *p); -------------------------------------------------------------------------------- /03_ring/03_singleThreadPktRing/recv_pkt.h: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int lcore_recv_pkt(struct rte_ring *in_r); 4 | -------------------------------------------------------------------------------- /03_ring/03_singleThreadPktRing/send_pkt.h: -------------------------------------------------------------------------------- 1 | 2 | #include "common.h" 3 | 4 | int lcore_send_pkt(struct lcore_params *p); -------------------------------------------------------------------------------- /02_send_recv/test/build/run-shared: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/02_send_recv/test/build/run-shared -------------------------------------------------------------------------------- /04_rss/multiThreadRing/portinit.h: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int port_init(uint16_t port, struct rte_mempool *mbuf_pool); -------------------------------------------------------------------------------- /02_send_recv/traffic_gen/portinit.h: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int port_init(uint16_t port, struct rte_mempool *mbuf_pool); -------------------------------------------------------------------------------- /03_ring/02_pktRing/build/run-shared: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/03_ring/02_pktRing/build/run-shared -------------------------------------------------------------------------------- /01_port_init/test/build/portinit-shared: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/01_port_init/test/build/portinit-shared -------------------------------------------------------------------------------- /03_ring/01_simpleRing/build/run-shared: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/03_ring/01_simpleRing/build/run-shared -------------------------------------------------------------------------------- /03_ring/03_singleThreadPktRing/portinit.h: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int port_init(uint16_t port, struct rte_mempool *mbuf_pool); -------------------------------------------------------------------------------- /04_rss/multiThreadRing/build/run-shared: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/04_rss/multiThreadRing/build/run-shared -------------------------------------------------------------------------------- /01_port_init/devinfo/build/devinfo-shared: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/01_port_init/devinfo/build/devinfo-shared -------------------------------------------------------------------------------- /02_send_recv/traffic_gen/build/run-shared: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/02_send_recv/traffic_gen/build/run-shared -------------------------------------------------------------------------------- /04_rss/multiThreadRing/ring.h: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int lcore_rx(struct lcore_params *p); 4 | int lcore_tx(struct lcore_params *p); 5 | -------------------------------------------------------------------------------- /03_ring/03_singleThreadPktRing/build/run-shared: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/03_ring/03_singleThreadPktRing/build/run-shared -------------------------------------------------------------------------------- /03_ring/03_singleThreadPktRing/ring.h: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int lcore_rx(struct lcore_params *p); 4 | int lcore_tx(struct rte_ring *tx_ring); 5 | -------------------------------------------------------------------------------- /a1_setup_mlx5_sriov_env/test/build/helloworld-shared: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zartbot/learn_dpdk/HEAD/a1_setup_mlx5_sriov_env/test/build/helloworld-shared -------------------------------------------------------------------------------- /fix_format.sh: -------------------------------------------------------------------------------- 1 | find ./ -name "*" | xargs dos2unix 2 | find ./ -name "*.c" | xargs sed -i 's/\t/ /g' 3 | find ./ -name "*.h" | xargs sed -i 's/\t/ /g' 4 | -------------------------------------------------------------------------------- /.vscode/c_cpp_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "Linux", 5 | "includePath": [ 6 | "${workspaceFolder}/**" 7 | ], 8 | "defines": [], 9 | "compilerPath": "/usr/bin/gcc", 10 | "cStandard": "c11", 11 | "cppStandard": "c++20", 12 | "intelliSenseMode": "gcc-x64", 13 | "compileCommands": "/home/zartbot/dpdk-21.02/build/compile_commands.json" 14 | } 15 | ], 16 | "version": 4 17 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.pythonPath": "/usr/bin/python3", 3 | "editor.rulers": [80, 120, 150], 4 | "files.watcherExclude": { 5 | "**/.git/objects/**": true, 6 | "**/.git/subtree-cache/**": true, 7 | "**/node_modules/*/**": true, 8 | "**/_note/*/**": true 9 | }, 10 | "editor.detectIndentation": false, //关闭检测第一个tab后面就tab 11 | "editor.renderControlCharacters": true, //制表符显示-> 12 | "editor.renderWhitespace": "all", //空格显示... 13 | "editor.tabSize": 4, //tab为四个空格 14 | "editor.insertSpaces": true //转为空格 15 | } 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Object files 5 | *.o 6 | *.ko 7 | *.obj 8 | *.elf 9 | 10 | # Linker output 11 | *.ilk 12 | *.map 13 | *.exp 14 | 15 | # Precompiled Headers 16 | *.gch 17 | *.pch 18 | 19 | # Libraries 20 | *.lib 21 | *.a 22 | *.la 23 | *.lo 24 | 25 | # Shared objects (inc. Windows DLLs) 26 | *.dll 27 | *.so 28 | *.so.* 29 | *.dylib 30 | 31 | # Executables 32 | *.exe 33 | *.out 34 | *.app 35 | *.i*86 36 | *.x86_64 37 | *.hex 38 | 39 | # Debug files 40 | *.dSYM/ 41 | *.su 42 | *.idb 43 | *.pdb 44 | 45 | # Kernel Module Compile Results 46 | *.mod* 47 | *.cmd 48 | .tmp_versions/ 49 | modules.order 50 | Module.symvers 51 | Mkfile.old 52 | dkms.conf 53 | 54 | */build* 55 | -------------------------------------------------------------------------------- /a3_crypto_qat/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## TODO 3 | 4 | 5 | sudo yum install libudev-devel //aka. systemd-devel 6 | 7 | 8 | add qat driver install, on both Broadwell and qat8950 9 | 10 | wget https://01.org/sites/default/files/downloads//qat1.7.l.4.12.0-00011.tar.gz 11 | mkdir qat 12 | cd qat 13 | tar vzxf ../qat1.7.l.4.12.0-00011.tar.gz 14 | ./configure 15 | make 16 | sudo make install 17 | 18 | 19 | ## add crypto support 20 | sudo yum install openssl-devel libpcap-devel libbpf-devel 21 | 22 | 23 | wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.gz 24 | tar vzxf nasm-2.15.05.tar.gz 25 | cd nasm-2.15.05 26 | ./configure 27 | make 28 | sudo make install 29 | 30 | 31 | git clone https://github.com/intel/intel-ipsec-mb 32 | cd intel-ipsec-mb 33 | make 34 | sudo make install 35 | 36 | 37 | -------------------------------------------------------------------------------- /03_ring/02_pktRing/common.h: -------------------------------------------------------------------------------- 1 | #ifndef __RING_COMMON_H_ 2 | #define __RING_COMMON_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | 17 | #define RX_RING_SIZE 2048 18 | #define TX_RING_SIZE 2048 19 | #define NUM_MBUFS ((64*1024)-1) 20 | #define MBUF_CACHE_SIZE 128 21 | #define SCHED_RX_RING_SZ 8192 22 | #define SCHED_TX_RING_SZ 65536 23 | 24 | #define NUM_RX_QUEUE 1 25 | #define NUM_TX_QUEUE 4 26 | 27 | #define BURST_SIZE 64 28 | #define BURST_SIZE_TX 32 29 | 30 | struct lcore_params { 31 | struct rte_ring *ring; 32 | struct rte_mempool *mem_pool; 33 | }; 34 | 35 | #endif -------------------------------------------------------------------------------- /03_ring/03_singleThreadPktRing/common.h: -------------------------------------------------------------------------------- 1 | #ifndef __RING_COMMON_H_ 2 | #define __RING_COMMON_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | 17 | #define RX_RING_SIZE 1024 18 | #define TX_RING_SIZE 1024 19 | #define NUM_MBUFS ((64*1024)-1) 20 | #define MBUF_CACHE_SIZE 128 21 | #define SCHED_RX_RING_SZ 8192 22 | #define SCHED_TX_RING_SZ 65536 23 | 24 | #define NUM_RX_QUEUE 1 25 | #define NUM_TX_QUEUE 1 26 | 27 | #define BURST_SIZE 64 28 | #define BURST_SIZE_TX 32 29 | 30 | struct lcore_params { 31 | struct rte_ring *rx_ring; 32 | struct rte_ring *tx_ring; 33 | struct rte_mempool *mem_pool; 34 | }; 35 | 36 | #endif -------------------------------------------------------------------------------- /04_rss/multiThreadRing/common.h: -------------------------------------------------------------------------------- 1 | #ifndef __RING_COMMON_H_ 2 | #define __RING_COMMON_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | 17 | #define RX_RING_SIZE 1024 18 | #define TX_RING_SIZE 1024 19 | #define NUM_MBUFS ((64*1024)-1) 20 | #define MBUF_CACHE_SIZE 128 21 | #define SCHED_RX_RING_SZ 8192 22 | #define SCHED_TX_RING_SZ 65536 23 | 24 | #define NUM_RX_QUEUE 2 25 | #define NUM_TX_QUEUE 8 26 | 27 | #define BURST_SIZE 64 28 | #define BURST_SIZE_TX 32 29 | 30 | struct lcore_params { 31 | struct rte_ring *rx_ring; 32 | struct rte_ring *tx_ring; 33 | uint16_t rx_queue_id; 34 | uint16_t tx_queue_id; 35 | struct rte_mempool *mem_pool; 36 | }; 37 | 38 | struct rx_params { 39 | uint16_t queue_id; 40 | struct rte_ring *ring; 41 | }; 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /03_ring/03_singleThreadPktRing/recv_pkt.c: -------------------------------------------------------------------------------- 1 | #include "recv_pkt.h" 2 | 3 | int lcore_recv_pkt(struct rte_ring *rx_ring) 4 | { 5 | 6 | struct rte_mbuf *bufs[BURST_SIZE]; 7 | printf("\nCore %u doing RX dequeue.\n", rte_lcore_id()); 8 | uint64_t freq = rte_get_tsc_hz(); 9 | 10 | uint64_t pkt_cnt = 0; 11 | uint64_t now = rte_rdtsc(); 12 | 13 | while (1) 14 | { 15 | 16 | const uint16_t nb_rx = rte_ring_dequeue_burst(rx_ring, 17 | (void *)bufs, BURST_SIZE, NULL); 18 | 19 | if (unlikely(nb_rx == 0)) 20 | { 21 | continue; 22 | } 23 | pkt_cnt += nb_rx; 24 | if (unlikely(rte_rdtsc() - now > freq)) 25 | { 26 | printf("PPS: %ld\n", pkt_cnt); 27 | pkt_cnt = 0; 28 | now = rte_rdtsc(); 29 | } 30 | uint16_t i; 31 | for (i = 0; i < nb_rx; i++) 32 | { 33 | rte_pktmbuf_free(bufs[i]); 34 | } 35 | 36 | } 37 | return 0; 38 | } 39 | -------------------------------------------------------------------------------- /03_ring/02_pktRing/recv_pkt.c: -------------------------------------------------------------------------------- 1 | #include "recv_pkt.h" 2 | 3 | int lcore_recv_pkt(struct rte_ring *rx_ring) 4 | { 5 | const int socket_id = rte_socket_id(); 6 | printf("\nCore %u doing RX dequeue.\n", rte_lcore_id()); 7 | uint64_t freq = rte_get_tsc_hz(); 8 | 9 | uint64_t pkt_cnt = 0; 10 | uint64_t now = rte_rdtsc(); 11 | 12 | while (1) 13 | { 14 | struct rte_mbuf *bufs[BURST_SIZE]; 15 | const uint16_t nb_rx = rte_ring_sc_dequeue_bulk(rx_ring, 16 | (void *)bufs, BURST_SIZE, NULL); 17 | 18 | if (unlikely(nb_rx == 0)) 19 | { 20 | continue; 21 | } 22 | pkt_cnt += nb_rx; 23 | if (unlikely(rte_rdtsc() - now > freq)) 24 | { 25 | printf("PPS: %ld\n", pkt_cnt); 26 | pkt_cnt = 0; 27 | now = rte_rdtsc(); 28 | } 29 | uint16_t i; 30 | for (i = 0; i < nb_rx; i++) 31 | { 32 | rte_pktmbuf_free(bufs[i]); 33 | } 34 | 35 | } 36 | return 0; 37 | } 38 | -------------------------------------------------------------------------------- /04_rss/multiThreadRing/recv_pkt.c: -------------------------------------------------------------------------------- 1 | #include "recv_pkt.h" 2 | 3 | int lcore_recv_pkt(struct rx_params *rx) 4 | { 5 | const int socket_id = rte_socket_id(); 6 | printf("Core %u doing RX dequeue.\n", rte_lcore_id()); 7 | uint64_t freq = rte_get_tsc_hz(); 8 | 9 | uint64_t pkt_cnt = 0; 10 | uint64_t now = rte_rdtsc(); 11 | 12 | while (1) 13 | { 14 | struct rte_mbuf *bufs[BURST_SIZE]; 15 | const uint16_t nb_rx = rte_ring_dequeue_burst(rx->ring, 16 | (void *)bufs, BURST_SIZE, NULL); 17 | 18 | if (unlikely(nb_rx == 0)) 19 | { 20 | continue; 21 | } 22 | pkt_cnt += nb_rx; 23 | if (unlikely(rte_rdtsc() - now > freq)) 24 | { 25 | printf("RX-Queue[%d] PPS: %ld\n",rx->queue_id, pkt_cnt); 26 | pkt_cnt = 0; 27 | now = rte_rdtsc(); 28 | } 29 | uint16_t i; 30 | for (i = 0; i < nb_rx; i++) 31 | { 32 | rte_pktmbuf_free(bufs[i]); 33 | } 34 | 35 | } 36 | return 0; 37 | } 38 | -------------------------------------------------------------------------------- /02_send_recv/traffic_gen/recv_pkt.c: -------------------------------------------------------------------------------- 1 | #include "recv_pkt.h" 2 | 3 | int lcore_recv_pkt(struct lcore_params *p) 4 | { 5 | const int socket_id = rte_socket_id(); 6 | printf("Core %u doing RX dequeue.\n", rte_lcore_id()); 7 | uint64_t freq = rte_get_tsc_hz(); 8 | 9 | uint64_t pkt_cnt = 0; 10 | uint64_t now = rte_rdtsc(); 11 | 12 | while (1) 13 | { 14 | struct rte_mbuf *bufs[BURST_SIZE]; 15 | const uint16_t nb_rx = rte_eth_rx_burst(0, p->rx_queue_id, bufs, 16 | BURST_SIZE); 17 | 18 | if (unlikely(rte_rdtsc() - now > freq)) 19 | { 20 | printf("RX-Queue[%d] PPS: %ld\n",p->rx_queue_id, pkt_cnt); 21 | pkt_cnt = 0; 22 | now = rte_rdtsc(); 23 | } 24 | if (unlikely(nb_rx == 0)) 25 | { 26 | continue; 27 | } 28 | pkt_cnt += nb_rx; 29 | 30 | uint16_t i; 31 | for (i = 0; i < nb_rx; i++) 32 | { 33 | rte_pktmbuf_free(bufs[i]); 34 | } 35 | 36 | } 37 | return 0; 38 | } 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Mie~~~ 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /a4_octeon/sdk_build.md: -------------------------------------------------------------------------------- 1 | export SDK_RELEASE=SDK-11.0.0.0 2 | mkdir ~/work-$SDK_RELEASE 3 | export SDK11_ROOT=~/work-$SDK_RELEASE 4 | 5 | cp base-SDK-11.0.0.0.tar.bz2 $SDK11_ROOT/. 6 | cd $SDK11_ROOT 7 | 8 | tar xf base-SDK-11.0.0.0.tar.bz2 9 | tar xf ../toolchain.tar.bz2 10 | 11 | cd toolchain/ 12 | tar xf marvell-tools-1013.0.tar.bz2 13 | cd $SDK11_ROOT 14 | 15 | export PATH=$SDK11_ROOT/toolchain/marvell-tools-1013.0/bin:$PATH 16 | 17 | cd $SDK11_ROOT/sdk-base 18 | ./scripts/ci/compile.sh -r $SDK_RELEASE cn96xx 19 | 20 | export SDK_RELEASE=SDK-11.0.0.0 21 | export SDK11_ROOT=~/work-$SDK_RELEASE 22 | export PATH=$SDK11_ROOT/toolchain/marvell-tools-1013.0/bin:$PATH 23 | cd $SDK11_ROOT/sdk-base 24 | ./scripts/ci/compile.sh -r $SDK_RELEASE cn96xx 25 | 26 | 27 | 28 | 29 | 30 | sudo mkfs.vfat -F32 -s 2 /dev/sdc1 31 | sudo mkfs.ext4 /dev/sdc2 32 | 33 | 34 | sudo mount /dev/sdc2 /mnt2 35 | sudo mount /dev/sdc1 /mnt1 36 | 37 | 38 | At the U-boot command line: 39 | 40 | 41 | 42 | usb start 43 | fatload usb 0:1 $loadaddr Image 44 | 45 | setenv bootargs 'setenv bootargs console=ttyAMA0,115200n8 earlycon=pl011,0x87e028000000 debug maxcpus=24 rootwait rw root=/dev/sda2 coherent_pool=16M' 46 | saveenv 47 | booti $loadaddr - $fdtcontroladdr 48 | 49 | 50 | 51 | 52 | export SDK_RELEASE=SDK-11.0.0.0 53 | export SDK11_ROOT=~/work-$SDK_RELEASE 54 | export PATH=$SDK11_ROOT/toolchain/marvell-tools-1013.0/bin:$PATH -------------------------------------------------------------------------------- /02_send_recv/traffic_gen/common.h: -------------------------------------------------------------------------------- 1 | #ifndef __RING_COMMON_H_ 2 | #define __RING_COMMON_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #define RX_RING_SIZE 1024 17 | #define TX_RING_SIZE 1024 18 | #define NUM_MBUFS ((64*1024)-1) 19 | #define MBUF_CACHE_SIZE 128 20 | #define SCHED_RX_RING_SZ 8192 21 | #define SCHED_TX_RING_SZ 65536 22 | 23 | #define NUM_RX_QUEUE 1 24 | #define NUM_TX_QUEUE 8 25 | 26 | #define BURST_SIZE 64 27 | #define BURST_SIZE_TX 32 28 | 29 | struct lcore_params { 30 | uint16_t rx_queue_id; 31 | uint16_t tx_queue_id; 32 | struct rte_mempool *mem_pool; 33 | }; 34 | 35 | static const struct rte_eth_conf port_conf_default = { 36 | .rxmode = { 37 | .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 38 | .mq_mode = ETH_MQ_RX_RSS, 39 | }, 40 | .rx_adv_conf = { 41 | .rss_conf = { 42 | .rss_key = NULL, 43 | //mlx5 support hf: 44 | .rss_hf = ETH_RSS_IP | ETH_RSS_TCP | ETH_RSS_UDP, 45 | //i40e support hf: 46 | //.rss_hf = ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV4_TCP, 47 | }, 48 | }, 49 | .txmode = { 50 | .mq_mode = ETH_MQ_TX_NONE, 51 | } 52 | 53 | }; 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /04_rss/multiThreadRing/ring.c: -------------------------------------------------------------------------------- 1 | #include "ring.h" 2 | 3 | int lcore_rx(struct lcore_params *p) 4 | { 5 | uint16_t port; 6 | struct rte_mbuf *bufs[BURST_SIZE * 2]; 7 | printf("Core %u doing packet RX.\n", rte_lcore_id()); 8 | 9 | port = 1; 10 | for (;;) 11 | { 12 | const uint16_t nb_rx = rte_eth_rx_burst(port, p->rx_queue_id, bufs, 13 | BURST_SIZE); 14 | if (unlikely(nb_rx == 0)) 15 | { 16 | continue; 17 | } 18 | uint16_t nb_ret = nb_rx; 19 | uint16_t sent = rte_ring_enqueue_burst(p->rx_ring, (void *)bufs, nb_ret, NULL); 20 | if (unlikely(sent < nb_ret)) 21 | { 22 | while (sent < nb_ret) 23 | rte_pktmbuf_free(bufs[sent++]); 24 | } 25 | } 26 | return 0; 27 | } 28 | 29 | int lcore_tx(struct lcore_params *p) 30 | { 31 | uint16_t port = 0; 32 | printf("Core %u doing packet TX.\n", rte_lcore_id()); 33 | struct rte_mbuf *bufs[BURST_SIZE_TX]; 34 | while (1) 35 | { 36 | const uint16_t nb_rx = rte_ring_dequeue_burst(p->tx_ring, 37 | (void *)bufs, BURST_SIZE_TX, NULL); 38 | if (unlikely(nb_rx == 0)) 39 | { 40 | 41 | continue; 42 | } 43 | unsigned int nb_tx = rte_eth_tx_burst(port, p->tx_queue_id, bufs, nb_rx); 44 | if (unlikely(nb_tx < nb_rx)) { 45 | do 46 | { 47 | rte_pktmbuf_free(bufs[nb_tx]); 48 | } while(++nb_tx < nb_rx); 49 | } 50 | } 51 | return 0; 52 | } 53 | -------------------------------------------------------------------------------- /01_port_init/test/main.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #define MAX_PORTS 16 15 | 16 | int main(int argc, char *argv[]) 17 | { 18 | 19 | int ret = rte_eal_init(argc, argv); 20 | if (ret < 0) 21 | rte_exit(EXIT_FAILURE, "initlize fail!"); 22 | 23 | printf("\n\n\n*****************************************\n"); 24 | 25 | int nb_ports; 26 | nb_ports = rte_eth_dev_count_avail(); 27 | printf("number of available port: %d\n", nb_ports); 28 | 29 | struct rte_eth_dev_info dev_info; 30 | for (int portid = 0; portid < nb_ports; ++portid) 31 | { 32 | ret = rte_eth_dev_info_get(portid, &dev_info); 33 | if (ret < 0) 34 | rte_exit(EXIT_FAILURE, "Cannot get device info: err=%d, port=%d\n", ret, portid); 35 | 36 | printf("port: %d Driver:%s\n", portid, dev_info.driver_name); 37 | } 38 | 39 | /* ethernet addresses of ports */ 40 | static struct rte_ether_addr ports_eth_addr[MAX_PORTS]; 41 | for (int portid = 0; portid < nb_ports; ++portid) 42 | { 43 | ret = rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); 44 | if (ret < 0) 45 | rte_exit(EXIT_FAILURE, "Cannot get MAC address: err=%d, port=%d\n", ret, portid); 46 | 47 | char mac[18]; 48 | rte_ether_format_addr(&mac[0], 18, &ports_eth_addr[portid]); 49 | printf("port: %d->MAC-> %s\n", portid, mac); 50 | } 51 | 52 | return 0; 53 | } -------------------------------------------------------------------------------- /03_ring/02_pktRing/main.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | #include "send_pkt.h" 9 | #include "recv_pkt.h" 10 | 11 | int main(int argc, char *argv[]) 12 | { 13 | 14 | struct rte_mempool *mbuf_pool; 15 | struct rte_ring *ring; 16 | struct lcore_params *lp; 17 | unsigned nb_ports = 2; 18 | uint16_t portid; 19 | 20 | int ret = rte_eal_init(argc, argv); 21 | if (ret < 0) 22 | rte_exit(EXIT_FAILURE, "initlize fail!"); 23 | 24 | /* Creates a new mempool in memory to hold the mbufs. */ 25 | mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports, 26 | MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 27 | 28 | if (mbuf_pool == NULL) 29 | rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 30 | 31 | ring = rte_ring_create("message_ring", SCHED_RX_RING_SZ, 32 | rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); 33 | if (ring == NULL) 34 | rte_exit(EXIT_FAILURE, "Cannot create ring\n"); 35 | 36 | unsigned int lcore_num = 1; 37 | 38 | /* Start pkt dequeue on port 1*/ 39 | rte_eal_remote_launch((lcore_function_t *)lcore_recv_pkt, ring, lcore_num++); 40 | 41 | /* start pkt enqueue on port 0*/ 42 | lp = rte_malloc(NULL, sizeof(*lp), 0); 43 | if (!lp) 44 | rte_panic("malloc failure\n"); 45 | *lp = (struct lcore_params){ring, mbuf_pool}; 46 | rte_eal_remote_launch((lcore_function_t *)lcore_send_pkt, lp, lcore_num ++); 47 | 48 | rte_eal_wait_lcore(lcore_num-1); 49 | return 0; 50 | } 51 | -------------------------------------------------------------------------------- /02_send_recv/test/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = run 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c 9 | 10 | # Build using pkg-config variables if possible 11 | ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) 12 | $(error "no installation of DPDK found") 13 | endif 14 | 15 | all: shared 16 | .PHONY: shared static 17 | shared: build/$(APP)-shared 18 | ln -sf $(APP)-shared build/$(APP) 19 | static: build/$(APP)-static 20 | ln -sf $(APP)-static build/$(APP) 21 | 22 | PKGCONF ?= pkg-config 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 $(shell $(PKGCONF) --cflags libdpdk) 26 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 27 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 28 | 29 | ifeq ($(MAKECMDGOALS),static) 30 | # check for broken pkg-config 31 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 32 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 33 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 34 | endif 35 | endif 36 | 37 | CFLAGS += -DALLOW_EXPERIMENTAL_API 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /03_ring/01_simpleRing/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = run 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c 9 | 10 | # Build using pkg-config variables if possible 11 | ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) 12 | $(error "no installation of DPDK found") 13 | endif 14 | 15 | all: shared 16 | .PHONY: shared static 17 | shared: build/$(APP)-shared 18 | ln -sf $(APP)-shared build/$(APP) 19 | static: build/$(APP)-static 20 | ln -sf $(APP)-static build/$(APP) 21 | 22 | PKGCONF ?= pkg-config 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 $(shell $(PKGCONF) --cflags libdpdk) 26 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 27 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 28 | 29 | ifeq ($(MAKECMDGOALS),static) 30 | # check for broken pkg-config 31 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 32 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 33 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 34 | endif 35 | endif 36 | 37 | CFLAGS += -DALLOW_EXPERIMENTAL_API 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /01_port_init/devinfo/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = devinfo 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c 9 | 10 | # Build using pkg-config variables if possible 11 | ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) 12 | $(error "no installation of DPDK found") 13 | endif 14 | 15 | all: shared 16 | .PHONY: shared static 17 | shared: build/$(APP)-shared 18 | ln -sf $(APP)-shared build/$(APP) 19 | static: build/$(APP)-static 20 | ln -sf $(APP)-static build/$(APP) 21 | 22 | PKGCONF ?= pkg-config 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 $(shell $(PKGCONF) --cflags libdpdk) 26 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 27 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 28 | 29 | ifeq ($(MAKECMDGOALS),static) 30 | # check for broken pkg-config 31 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 32 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 33 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 34 | endif 35 | endif 36 | 37 | CFLAGS += -DALLOW_EXPERIMENTAL_API 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /01_port_init/test/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = portinit 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c 9 | 10 | # Build using pkg-config variables if possible 11 | ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) 12 | $(error "no installation of DPDK found") 13 | endif 14 | 15 | all: shared 16 | .PHONY: shared static 17 | shared: build/$(APP)-shared 18 | ln -sf $(APP)-shared build/$(APP) 19 | static: build/$(APP)-static 20 | ln -sf $(APP)-static build/$(APP) 21 | 22 | PKGCONF ?= pkg-config 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 $(shell $(PKGCONF) --cflags libdpdk) 26 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 27 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 28 | 29 | ifeq ($(MAKECMDGOALS),static) 30 | # check for broken pkg-config 31 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 32 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 33 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 34 | endif 35 | endif 36 | 37 | CFLAGS += -DALLOW_EXPERIMENTAL_API 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /a1_setup_mlx5_sriov_env/test/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = helloworld 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c 9 | 10 | # Build using pkg-config variables if possible 11 | ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) 12 | $(error "no installation of DPDK found") 13 | endif 14 | 15 | all: shared 16 | .PHONY: shared static 17 | shared: build/$(APP)-shared 18 | ln -sf $(APP)-shared build/$(APP) 19 | static: build/$(APP)-static 20 | ln -sf $(APP)-static build/$(APP) 21 | 22 | PKGCONF ?= pkg-config 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 $(shell $(PKGCONF) --cflags libdpdk) 26 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 27 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 28 | 29 | ifeq ($(MAKECMDGOALS),static) 30 | # check for broken pkg-config 31 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 32 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 33 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 34 | endif 35 | endif 36 | 37 | CFLAGS += -DALLOW_EXPERIMENTAL_API 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /03_ring/02_pktRing/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = run 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c send_pkt.c recv_pkt.c 9 | 10 | # Build using pkg-config variables if possible 11 | ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) 12 | $(error "no installation of DPDK found") 13 | endif 14 | 15 | all: shared 16 | .PHONY: shared static 17 | shared: build/$(APP)-shared 18 | ln -sf $(APP)-shared build/$(APP) 19 | static: build/$(APP)-static 20 | ln -sf $(APP)-static build/$(APP) 21 | 22 | PKGCONF ?= pkg-config 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 $(shell $(PKGCONF) --cflags libdpdk) 26 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 27 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 28 | 29 | ifeq ($(MAKECMDGOALS),static) 30 | # check for broken pkg-config 31 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 32 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 33 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 34 | endif 35 | endif 36 | 37 | CFLAGS += -DALLOW_EXPERIMENTAL_API 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /03_ring/03_singleThreadPktRing/ring.c: -------------------------------------------------------------------------------- 1 | #include "ring.h" 2 | 3 | int lcore_rx(struct lcore_params *p) 4 | { 5 | const int socket_id = rte_socket_id(); 6 | uint16_t port; 7 | struct rte_mbuf *bufs[BURST_SIZE * 2]; 8 | printf("\nCore %u doing packet RX.\n", rte_lcore_id()); 9 | 10 | port = 1; 11 | for (;;) 12 | { 13 | const uint16_t nb_rx = rte_eth_rx_burst(port, 0, bufs, 14 | BURST_SIZE); 15 | if (unlikely(nb_rx == 0)) 16 | { 17 | continue; 18 | } 19 | 20 | uint16_t nb_ret = nb_rx; 21 | uint16_t sent = rte_ring_sp_enqueue_bulk(p->rx_ring, (void *)bufs, nb_ret, NULL); 22 | if (unlikely(sent < nb_ret)) 23 | { 24 | while (sent < nb_ret) 25 | rte_pktmbuf_free(bufs[sent++]); 26 | } 27 | } 28 | return 0; 29 | } 30 | 31 | int lcore_tx(struct rte_ring *ring) 32 | { 33 | const int socket_id = rte_socket_id(); 34 | uint16_t port = 0; 35 | printf("\nCore %u doing packet TX.\n", rte_lcore_id()); 36 | while (1) 37 | { 38 | struct rte_mbuf *bufs[BURST_SIZE_TX]; 39 | const uint16_t nb_rx = rte_ring_sc_dequeue_bulk(ring, 40 | (void *)bufs, BURST_SIZE_TX, NULL); 41 | if (unlikely(nb_rx == 0)) 42 | continue; 43 | 44 | /* for traffic we receive, queue it up for transmit */ 45 | unsigned int nb_tx = rte_eth_tx_burst(0, 0,bufs, nb_rx); 46 | if (unlikely(nb_tx < nb_rx)) { 47 | do 48 | { 49 | rte_pktmbuf_free(bufs[nb_tx]); 50 | 51 | } while(++nb_tx < nb_rx); 52 | } 53 | } 54 | return 0; 55 | } 56 | -------------------------------------------------------------------------------- /02_send_recv/traffic_gen/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = run 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c portinit.c send_pkt.c recv_pkt.c 9 | 10 | # Build using pkg-config variables if possible 11 | ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) 12 | $(error "no installation of DPDK found") 13 | endif 14 | 15 | all: shared 16 | .PHONY: shared static 17 | shared: build/$(APP)-shared 18 | ln -sf $(APP)-shared build/$(APP) 19 | static: build/$(APP)-static 20 | ln -sf $(APP)-static build/$(APP) 21 | 22 | PKGCONF ?= pkg-config 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 $(shell $(PKGCONF) --cflags libdpdk) 26 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 27 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 28 | 29 | ifeq ($(MAKECMDGOALS),static) 30 | # check for broken pkg-config 31 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 32 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 33 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 34 | endif 35 | endif 36 | 37 | CFLAGS += -DALLOW_EXPERIMENTAL_API 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /04_rss/multiThreadRing/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = run 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c portinit.c send_pkt.c recv_pkt.c ring.c 9 | 10 | # Build using pkg-config variables if possible 11 | ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) 12 | $(error "no installation of DPDK found") 13 | endif 14 | 15 | all: shared 16 | .PHONY: shared static 17 | shared: build/$(APP)-shared 18 | ln -sf $(APP)-shared build/$(APP) 19 | static: build/$(APP)-static 20 | ln -sf $(APP)-static build/$(APP) 21 | 22 | PKGCONF ?= pkg-config 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 $(shell $(PKGCONF) --cflags libdpdk) 26 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 27 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 28 | 29 | ifeq ($(MAKECMDGOALS),static) 30 | # check for broken pkg-config 31 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 32 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 33 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 34 | endif 35 | endif 36 | 37 | CFLAGS += -DALLOW_EXPERIMENTAL_API 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /03_ring/03_singleThreadPktRing/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = run 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c portinit.c send_pkt.c recv_pkt.c ring.c 9 | 10 | # Build using pkg-config variables if possible 11 | ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) 12 | $(error "no installation of DPDK found") 13 | endif 14 | 15 | all: shared 16 | .PHONY: shared static 17 | shared: build/$(APP)-shared 18 | ln -sf $(APP)-shared build/$(APP) 19 | static: build/$(APP)-static 20 | ln -sf $(APP)-static build/$(APP) 21 | 22 | PKGCONF ?= pkg-config 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 $(shell $(PKGCONF) --cflags libdpdk) 26 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 27 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 28 | 29 | ifeq ($(MAKECMDGOALS),static) 30 | # check for broken pkg-config 31 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 32 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 33 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 34 | endif 35 | endif 36 | 37 | CFLAGS += -DALLOW_EXPERIMENTAL_API 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /02_send_recv/traffic_gen/main.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | #include "portinit.h" 9 | #include "send_pkt.h" 10 | #include "recv_pkt.h" 11 | 12 | int main(int argc, char *argv[]) 13 | { 14 | struct rte_mempool *mbuf_pool; 15 | struct rte_ring *tx_ring[NUM_TX_QUEUE]; 16 | struct rte_ring *rx_ring[NUM_RX_QUEUE]; 17 | struct lcore_params *lp_tx[NUM_TX_QUEUE]; 18 | struct lcore_params *lp_rx[NUM_RX_QUEUE]; 19 | struct rx_params *rxp[NUM_RX_QUEUE]; 20 | uint16_t portid; 21 | 22 | int ret = rte_eal_init(argc, argv); 23 | if (ret < 0) 24 | rte_exit(EXIT_FAILURE, "initlize fail!"); 25 | 26 | /* Creates a new mempool in memory to hold the mbufs. */ 27 | mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS, 28 | MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 29 | 30 | if (mbuf_pool == NULL) 31 | rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 32 | 33 | if (port_init(0, mbuf_pool) != 0) 34 | rte_exit(EXIT_FAILURE, "Cannot init port 0\n"); 35 | 36 | unsigned int lcore_num = 1; 37 | 38 | /* Start rx core */ 39 | for (int i = 0; i < NUM_RX_QUEUE; ++i) 40 | { 41 | lp_rx[i] = rte_malloc(NULL, sizeof(*lp_rx[i]), 0); 42 | if (!lp_rx[i]) 43 | rte_panic("malloc failure\n"); 44 | *lp_rx[i] = (struct lcore_params){i, i, mbuf_pool}; 45 | rte_eal_remote_launch((lcore_function_t *)lcore_recv_pkt, lp_rx[i], lcore_num++); 46 | } 47 | 48 | 49 | /* Start tx core */ 50 | for (int i = 0; i < NUM_TX_QUEUE; ++i) 51 | { 52 | lp_tx[i] = rte_malloc(NULL, sizeof(*lp_tx[i]), 0); 53 | if (!lp_tx[i]) 54 | rte_panic("malloc failure\n"); 55 | *lp_tx[i] = (struct lcore_params){i, i, mbuf_pool}; 56 | rte_eal_remote_launch((lcore_function_t *)lcore_send_pkt, lp_tx[i], lcore_num++); 57 | } 58 | 59 | 60 | rte_eal_wait_lcore(lcore_num - 1); 61 | return 0; 62 | } 63 | -------------------------------------------------------------------------------- /03_ring/03_singleThreadPktRing/main.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | #include "portinit.h" 9 | #include "send_pkt.h" 10 | #include "recv_pkt.h" 11 | #include "ring.h" 12 | 13 | int main(int argc, char *argv[]) 14 | { 15 | 16 | struct rte_mempool *mbuf_pool; 17 | struct rte_ring *tx_ring; 18 | struct rte_ring *rx_ring; 19 | struct lcore_params *lp; 20 | unsigned nb_ports = 2; 21 | uint16_t portid; 22 | 23 | int ret = rte_eal_init(argc, argv); 24 | if (ret < 0) 25 | rte_exit(EXIT_FAILURE, "initlize fail!"); 26 | 27 | /* Creates a new mempool in memory to hold the mbufs. */ 28 | mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports, 29 | MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 30 | 31 | if (mbuf_pool == NULL) 32 | rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 33 | 34 | tx_ring = rte_ring_create("output_ring", SCHED_TX_RING_SZ, 35 | rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); 36 | if (tx_ring == NULL) 37 | rte_exit(EXIT_FAILURE, "Cannot create output ring\n"); 38 | 39 | rx_ring = rte_ring_create("Input_ring", SCHED_RX_RING_SZ, 40 | rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); 41 | if (rx_ring == NULL) 42 | rte_exit(EXIT_FAILURE, "Cannot create output ring\n"); 43 | 44 | /* Initialize all ports. */ 45 | RTE_ETH_FOREACH_DEV(portid) 46 | if (port_init(portid, mbuf_pool) != 0) 47 | rte_exit(EXIT_FAILURE, "Cannot init port %" PRIu16 "\n", 48 | portid); 49 | 50 | unsigned int lcore_num = 1; 51 | /* Start tx core */ 52 | rte_eal_remote_launch((lcore_function_t *)lcore_tx, tx_ring, lcore_num++); 53 | 54 | /* Start rx core */ 55 | struct lcore_params *pr = rte_malloc(NULL, sizeof(*pr), 0); 56 | if (!pr) 57 | rte_panic("malloc failure\n"); 58 | *pr = (struct lcore_params){rx_ring, tx_ring, mbuf_pool}; 59 | rte_eal_remote_launch((lcore_function_t *)lcore_rx, pr, lcore_num++); 60 | 61 | /* Start pkt dequeue on port 1*/ 62 | rte_eal_remote_launch((lcore_function_t *)lcore_recv_pkt, rx_ring, lcore_num++); 63 | 64 | /* start pkt enqueue on port 0*/ 65 | lp = rte_malloc(NULL, sizeof(*lp), 0); 66 | if (!lp) 67 | rte_panic("malloc failure\n"); 68 | *lp = (struct lcore_params){rx_ring, tx_ring, mbuf_pool}; 69 | rte_eal_remote_launch((lcore_function_t *)lcore_send_pkt, lp, lcore_num++); 70 | 71 | rte_eal_wait_lcore(lcore_num - 1); 72 | return 0; 73 | } 74 | -------------------------------------------------------------------------------- /03_ring/01_simpleRing/main.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #define RTE_LOGTYPE_APP RTE_LOGTYPE_USER1 26 | 27 | static const char *_MSG_POOL = "MSG_POOL"; 28 | static const char *_Z_RING = "ZARTBOT_RING"; 29 | 30 | struct rte_ring *send_ring, *zartbot_ring; 31 | struct rte_mempool *message_pool; 32 | volatile int quit = 0; 33 | 34 | static int 35 | lcore_recv(__rte_unused void *arg) 36 | { 37 | unsigned lcore_id = rte_lcore_id(); 38 | printf("Starting core %u\n", lcore_id); 39 | while (!quit){ 40 | void *msg; 41 | if (rte_ring_dequeue(zartbot_ring, &msg) < 0){ 42 | usleep(5); 43 | continue; 44 | } 45 | printf("core %u: Received '%s'\n", lcore_id, (char *)msg); 46 | rte_mempool_put(message_pool, msg); 47 | } 48 | return 0; 49 | } 50 | 51 | static int 52 | lcore_send(__rte_unused void *arg) 53 | { 54 | unsigned lcore_id = rte_lcore_id(); 55 | struct rte_ring *tx_ring; 56 | printf("Starting core %u\n", lcore_id); 57 | void *msg = NULL; 58 | if (rte_mempool_get(message_pool, &msg) < 0) 59 | rte_panic("Failed to get message buffer\n"); 60 | 61 | for (int i=0;i<100;i++) { 62 | strlcpy((char *)msg, "hello", 128); 63 | if (rte_ring_enqueue(zartbot_ring, msg) < 0) { 64 | printf("Failed to send message - message discarded\n"); 65 | rte_mempool_put(message_pool, msg); 66 | } 67 | sleep(1); 68 | } 69 | } 70 | 71 | int 72 | main(int argc, char **argv) 73 | { 74 | const unsigned flags = 0; 75 | const unsigned ring_size = 1024*32; 76 | const unsigned pool_size = 1023; 77 | const unsigned pool_cache = 32; 78 | const unsigned priv_data_sz = 0; 79 | 80 | int ret; 81 | unsigned lcore_id; 82 | 83 | ret = rte_eal_init(argc, argv); 84 | if (ret < 0) 85 | rte_exit(EXIT_FAILURE, "Cannot init EAL\n"); 86 | 87 | zartbot_ring = rte_ring_create(_Z_RING, ring_size, rte_socket_id(), flags); 88 | message_pool = rte_mempool_create(_MSG_POOL, pool_size, 89 | 128, pool_cache, priv_data_sz, 90 | NULL, NULL, NULL, NULL, 91 | rte_socket_id(), flags); 92 | 93 | if (zartbot_ring == NULL) 94 | rte_exit(EXIT_FAILURE, "Problem getting receiving ring\n"); 95 | if (message_pool == NULL) 96 | rte_exit(EXIT_FAILURE, "Problem getting message pool\n"); 97 | 98 | RTE_LOG(INFO, APP, "Finished Process Init.\n"); 99 | 100 | RTE_LCORE_FOREACH_WORKER(lcore_id) { 101 | if (lcore_id == 1) { 102 | rte_eal_remote_launch(lcore_send, NULL, lcore_id); 103 | } 104 | if (lcore_id == 2) { 105 | rte_eal_remote_launch(lcore_recv, NULL, lcore_id); 106 | } 107 | 108 | } 109 | 110 | rte_eal_mp_wait_lcore(); 111 | return 0; 112 | } 113 | -------------------------------------------------------------------------------- /04_rss/multiThreadRing/main.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | #include "portinit.h" 9 | #include "send_pkt.h" 10 | #include "recv_pkt.h" 11 | #include "ring.h" 12 | 13 | int main(int argc, char *argv[]) 14 | { 15 | struct rte_mempool *mbuf_pool; 16 | struct rte_ring *tx_ring[NUM_TX_QUEUE]; 17 | struct rte_ring *rx_ring[NUM_RX_QUEUE]; 18 | struct lcore_params *lp_tx[NUM_TX_QUEUE]; 19 | struct lcore_params *lp_rx[NUM_RX_QUEUE]; 20 | struct rx_params *rxp[NUM_RX_QUEUE]; 21 | unsigned nb_ports = 2; 22 | uint16_t portid; 23 | 24 | int ret = rte_eal_init(argc, argv); 25 | if (ret < 0) 26 | rte_exit(EXIT_FAILURE, "initlize fail!"); 27 | 28 | /* Creates a new mempool in memory to hold the mbufs. */ 29 | mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports, 30 | MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 31 | 32 | if (mbuf_pool == NULL) 33 | rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 34 | 35 | for (int i = 0; i < NUM_TX_QUEUE; ++i) 36 | { 37 | char tx_ring_name[14]; 38 | sprintf(tx_ring_name, "output_ring%d", i); 39 | tx_ring[i] = rte_ring_create(tx_ring_name, SCHED_TX_RING_SZ, 40 | rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); 41 | if (tx_ring[i] == NULL) 42 | rte_exit(EXIT_FAILURE, "Cannot create output ring\n"); 43 | } 44 | 45 | for (int i = 0; i < NUM_RX_QUEUE; ++i) 46 | { 47 | char rx_ring_name[14]; 48 | sprintf(rx_ring_name, "input_ring%d", i); 49 | rx_ring[i] = rte_ring_create(rx_ring_name, SCHED_RX_RING_SZ, 50 | rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); 51 | if (rx_ring[i] == NULL) 52 | rte_exit(EXIT_FAILURE, "Cannot create input ring\n"); 53 | } 54 | 55 | /* Initialize all ports. */ 56 | RTE_ETH_FOREACH_DEV(portid) 57 | if (port_init(portid, mbuf_pool) != 0) 58 | rte_exit(EXIT_FAILURE, "Cannot init port %" PRIu16 "\n", 59 | portid); 60 | 61 | unsigned int lcore_num = 1; 62 | 63 | /* Start rx core */ 64 | for (int i = 0; i < NUM_RX_QUEUE; ++i) 65 | { 66 | lp_rx[i] = rte_malloc(NULL, sizeof(*lp_rx[i]), 0); 67 | if (!lp_rx[i]) 68 | rte_panic("malloc failure\n"); 69 | *lp_rx[i] = (struct lcore_params){rx_ring[i], NULL, i, i, mbuf_pool}; 70 | rte_eal_remote_launch((lcore_function_t *)lcore_rx, lp_rx[i], lcore_num++); 71 | //start pkt dequeue process 72 | rxp[i] = rte_malloc(NULL, sizeof(*rxp[i]), 0); 73 | if (!rxp[i]) 74 | rte_panic("malloc failure\n"); 75 | *rxp[i] = (struct rx_params){i,rx_ring[i]}; 76 | rte_eal_remote_launch((lcore_function_t *)lcore_recv_pkt, rxp[i], lcore_num++); 77 | } 78 | 79 | 80 | /* Start tx core */ 81 | for (int i = 0; i < NUM_TX_QUEUE; ++i) 82 | { 83 | lp_tx[i] = rte_malloc(NULL, sizeof(*lp_tx[i]), 0); 84 | if (!lp_tx[i]) 85 | rte_panic("malloc failure\n"); 86 | *lp_tx[i] = (struct lcore_params){NULL, tx_ring[i], i, i, mbuf_pool}; 87 | rte_eal_remote_launch((lcore_function_t *)lcore_tx, lp_tx[i], lcore_num++); 88 | //start packet enqueue 89 | rte_eal_remote_launch((lcore_function_t *)lcore_send_pkt, lp_tx[i], lcore_num++); 90 | } 91 | 92 | 93 | rte_eal_wait_lcore(lcore_num - 1); 94 | return 0; 95 | } 96 | -------------------------------------------------------------------------------- /a1_setup_mlx5_sriov_env/sriov.md: -------------------------------------------------------------------------------- 1 | 2 | ## 配置SRIOV 3 | 4 | 改到`root`登录 5 | ```bash 6 | su root 7 | 8 | [root@netdev test]# mst start 9 | Starting MST (Mellanox Software Tools) driver set 10 | Loading MST PCI module - Success 11 | Loading MST PCI configuration module - Success 12 | Create devices 13 | Unloading MST PCI module (unused) - Success 14 | ``` 15 | 16 | 查看device-id 17 | ```bash 18 | [root@netdev test]# mst status 19 | MST modules: 20 | ------------ 21 | MST PCI module is not loaded 22 | MST PCI configuration module loaded 23 | 24 | MST devices: 25 | ------------ 26 | /dev/mst/mt4121_pciconf0 - PCI configuration cycles access. 27 | domain:bus:dev.fn=0000:86:00.0 addr.reg=88 data.reg=92 cr_bar.gw_offset=-1 28 | Chip revision is: 00 29 | ``` 30 | 此步主要是获取`/dev/mst/mt4121_pciconf0` 31 | 32 | 然后检查相关的配置 33 | ```bash 34 | [root@netdev test]# mlxconfig -d /dev/mst/mt4121_pciconf0 q | egrep -e "SRIOV|NUM_OF_VF|UCTX" 35 | NUM_OF_VFS 4 36 | SRIOV_EN True(1) 37 | UCTX_EN False(0) 38 | SRIOV_IB_ROUTING_MODE_P1 LID(1) 39 | SRIOV_IB_ROUTING_MODE_P2 LID(1) 40 | ``` 41 | 需要将NUM_OF_VFS设置为4个、然后Enable `SRIOV_EN`和`UCTX_EN` 42 | 43 | ```bash 44 | [root@netdev test]# mlxconfig -d /dev/mst/mt4121_pciconf0 set SRIOV_EN=1 NUM_OF_VFS=4 UCTX_EN=1 45 | 46 | Device #1: 47 | ---------- 48 | 49 | Device type: ConnectX5 50 | Name: MCX516A-CDA_Ax_Bx 51 | Description: ConnectX-5 Ex EN network interface card; 100GbE dual-port QSFP28; PCIe4.0 x16; tall bracket; ROHS R6 52 | Device: /dev/mst/mt4121_pciconf0 53 | 54 | Configurations: Next Boot New 55 | SRIOV_EN True(1) True(1) 56 | NUM_OF_VFS 4 4 57 | UCTX_EN False(0) True(1) 58 | 59 | Apply new Configuration? (y/n) [n] : y 60 | Applying... Done! 61 | -I- Please reboot machine to load new configurations. 62 | ``` 63 | 64 | 配置完成后重启整机 65 | ```bash 66 | root 67 | ``` 68 | 69 | ### 创建VF 70 | 71 | 查看接口 72 | ``` 73 | [zartbot@netdev ~]$ sudo ibdev2netdev 74 | mlx5_0 port 1 ==> ens17f0 (Up) 75 | mlx5_1 port 1 ==> ens17f1 (Up) 76 | ``` 77 | 78 | 查看最多能够支持的VF数量 79 | ``` 80 | [zartbot@netdev ~]$ cat /sys/class/net/ens17f0/device/sriov_totalvfs 81 | 4 82 | [zartbot@netdev ~]$ cat /sys/class/net/ens17f1/device/sriov_totalvfs 83 | 4 84 | ``` 85 | 启用VF 86 | ``` 87 | echo 4 | sudo tee /sys/class/net/ens17f1/device/sriov_numvfs 88 | echo 4 | sudo tee /sys/class/net/ens17f0/device/sriov_numvfs 89 | ``` 90 | 然后lspci就可以看到网卡了 91 | 92 | ```bash 93 | [zartbot@netdev ~]$ lspci | grep Mellanox 94 | 86:00.0 Ethernet controller: Mellanox Technologies MT28800 Family [ConnectX-5 Ex] 95 | 86:00.1 Ethernet controller: Mellanox Technologies MT28800 Family [ConnectX-5 Ex] 96 | 86:00.2 Ethernet controller: Mellanox Technologies MT28800 Family [ConnectX-5 Ex Virtual Function] 97 | 86:00.3 Ethernet controller: Mellanox Technologies MT28800 Family [ConnectX-5 Ex Virtual Function] 98 | 86:00.4 Ethernet controller: Mellanox Technologies MT28800 Family [ConnectX-5 Ex Virtual Function] 99 | 86:00.5 Ethernet controller: Mellanox Technologies MT28800 Family [ConnectX-5 Ex Virtual Function] 100 | 86:00.6 Ethernet controller: Mellanox Technologies MT28800 Family [ConnectX-5 Ex Virtual Function] 101 | 86:00.7 Ethernet controller: Mellanox Technologies MT28800 Family [ConnectX-5 Ex Virtual Function] 102 | 86:01.0 Ethernet controller: Mellanox Technologies MT28800 Family [ConnectX-5 Ex Virtual Function] 103 | 86:01.1 Ethernet controller: Mellanox Technologies MT28800 Family [ConnectX-5 Ex Virtual Function] 104 | ``` 105 | -------------------------------------------------------------------------------- /03_ring/README2.md: -------------------------------------------------------------------------------- 1 | ## 利用Ring收发报文 2 | 3 | 这个例子源文件在`02_pktRing`中,我们只创建了一个Ring,然后分别在两个不同的Core上调用收发函数 4 | ```c 5 | ring = rte_ring_create("message_ring", SCHED_RX_RING_SZ, 6 | rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); 7 | /* Start pkt dequeue on port 1*/ 8 | rte_eal_remote_launch((lcore_function_t *)lcore_recv_pkt, ring, lcore_num++); 9 | 10 | /* start pkt enqueue on port 0*/ 11 | lp = rte_malloc(NULL, sizeof(*lp), 0); 12 | if (!lp) 13 | rte_panic("malloc failure\n"); 14 | *lp = (struct lcore_params){ring, mbuf_pool}; 15 | rte_eal_remote_launch((lcore_function_t *)lcore_send_pkt, lp, lcore_num ++); 16 | ``` 17 | 18 | 大概就是如下这样 19 | ```bash 20 | lcore2(lcore_send_pkt)---->ring--->lcore1(lcore_recv_pkt) 21 | ``` 22 | 23 | 发包函数如下， 主要是`rte_ring_sp_enqueue_bulk`函数 24 | ```c 25 | 26 | int lcore_send_pkt(struct lcore_params *p) 27 | { 28 | //此处省略.... 29 | struct rte_mbuf *pkts[BURST_SIZE]; 30 | 31 | for (;;) 32 | { 33 | //批量allocate mbuf 34 | rte_pktmbuf_alloc_bulk(p->mem_pool, pkts, BURST_SIZE); 35 | for (int i = 0; i < BURST_SIZE; i++) 36 | { 37 | //修改以太网头 38 | eth_hdr = rte_pktmbuf_mtod(pkts[i], struct rte_ether_hdr *); 39 | eth_hdr->d_addr = d_addr; 40 | eth_hdr->s_addr = s_addr; 41 | eth_hdr->ether_type = ether_type; 42 | 43 | //此处省略.... 44 | } 45 | 46 | //发包函数 47 | uint16_t nb_send = BURST_SIZE; 48 | uint16_t sent = rte_ring_sp_enqueue_bulk(p->ring, (void *)pkts, nb_send, NULL); 49 | 50 | //没有发出去的包需要Free掉 51 | if (unlikely(sent < nb_send)) 52 | { 53 | while (sent < nb_send) 54 | { 55 | rte_pktmbuf_free(pkts[sent++]); 56 | } 57 | } 58 | } 59 | return 0; 60 | } 61 | ``` 62 | 63 | 收包函数和以前一样,计算pps. 64 | 65 | ## 单线程以太网收发包 66 | 67 | 在前一个示例的基础上，我们改成如下这样，代码在`03_singleThreadPktRing`内 68 | ```bash 69 | lcore_send_pkt-->tx_ring-->lcore_tx--Port0-->Port1-->lcore_rx-->lcore_recv_pkt 70 | ``` 71 | 主要是增加了两个额外的函数,接收时，从`rte_eth_rx_burst`收包，然后`rte_ring_sp_enqueue_bulk` 72 | ```c 73 | 74 | int lcore_rx(struct lcore_params *p) 75 | { 76 | const int socket_id = rte_socket_id(); 77 | uint16_t port; 78 | struct rte_mbuf *bufs[BURST_SIZE * 2]; 79 | printf("\nCore %u doing packet RX.\n", rte_lcore_id()); 80 | 81 | port = 1; 82 | for (;;) 83 | { 84 | const uint16_t nb_rx = rte_eth_rx_burst(port, 0, bufs, 85 | BURST_SIZE); 86 | if (unlikely(nb_rx == 0)) 87 | { 88 | continue; 89 | } 90 | 91 | uint16_t nb_ret = nb_rx; 92 | uint16_t sent = rte_ring_sp_enqueue_bulk(p->rx_ring, (void *)bufs, nb_ret, NULL); 93 | if (unlikely(sent < nb_ret)) 94 | { 95 | while (sent < nb_ret) 96 | rte_pktmbuf_free(bufs[sent++]); 97 | } 98 | } 99 | return 0; 100 | } 101 | ``` 102 | 发送时，从`rte_ring_sc_dequeue_bulk`收包，然后`rte_eth_tx_burst`网卡发出去 103 | ```c 104 | 105 | int lcore_tx(struct rte_ring *ring) 106 | { 107 | const int socket_id = rte_socket_id(); 108 | uint16_t port = 0; 109 | printf("\nCore %u doing packet TX.\n", rte_lcore_id()); 110 | while (1) 111 | { 112 | struct rte_mbuf *bufs[BURST_SIZE_TX]; 113 | const uint16_t nb_rx = rte_ring_sc_dequeue_bulk(ring, 114 | (void *)bufs, BURST_SIZE_TX, NULL); 115 | if (unlikely(nb_rx == 0)) 116 | continue; 117 | 118 | /* for traffic we receive, queue it up for transmit */ 119 | unsigned int nb_tx = rte_eth_tx_burst(0, 0,bufs, nb_rx); 120 | if (unlikely(nb_tx < nb_rx)) { 121 | do 122 | { 123 | rte_pktmbuf_free(bufs[nb_tx]); 124 | 125 | } while(++nb_tx < nb_rx); 126 | } 127 | } 128 | return 0; 129 | } 130 | ``` 131 | 132 | 133 | 134 | 135 | -------------------------------------------------------------------------------- /02_send_recv/traffic_gen/send_pkt.c: -------------------------------------------------------------------------------- 1 | 2 | #include "send_pkt.h" 3 | 4 | rte_be32_t string_to_ip(char *s) 5 | { 6 | unsigned char a[4]; 7 | int rc = sscanf(s, "%hhd.%hhd.%hhd.%hhd", a + 0, a + 1, a + 2, a + 3); 8 | if (rc != 4) 9 | { 10 | fprintf(stderr, "bad source IP address format. Use like: 1.1.1.1\n"); 11 | exit(1); 12 | } 13 | return (rte_be32_t)(a[3]) << 24 | 14 | (rte_be32_t)(a[2]) << 16 | 15 | (rte_be32_t)(a[1]) << 8 | 16 | (rte_be32_t)(a[0]); 17 | } 18 | 19 | 20 | int lcore_send_pkt(struct lcore_params *p) 21 | { 22 | const int socket_id = rte_socket_id(); 23 | printf("Core %u doing packet enqueue.\n", rte_lcore_id()); 24 | 25 | struct rte_ether_hdr *eth_hdr; 26 | struct rte_ipv4_hdr *ipv4_hdr; 27 | struct rte_udp_hdr *udp_hdr; 28 | 29 | //init mac 30 | struct rte_ether_addr s_addr = {{0xec, 0x0d, 0x9a, 0xc5, 0xdf, 0xfc}}; 31 | struct rte_ether_addr d_addr = {{0xca, 0xfe, 0xbe, 0xef, 0x00, 0x01}}; 32 | 33 | //init IP header 34 | rte_be32_t s_ip_addr = string_to_ip("1.0.0.253"); 35 | rte_be32_t d_ip_addr = string_to_ip("1.0.0.1"); 36 | uint16_t ether_type = rte_cpu_to_be_16(0x0800); 37 | //Defined header in UDP 38 | struct SRoU 39 | { 40 | uint8_t magic_num; 41 | uint8_t srou_length; 42 | uint8_t flags; 43 | uint8_t next_protcol; 44 | uint64_t pad; 45 | }; 46 | 47 | struct rte_mbuf *pkts[BURST_SIZE]; 48 | 49 | for (;;) 50 | { 51 | rte_pktmbuf_alloc_bulk(p->mem_pool, pkts, BURST_SIZE); 52 | 53 | uint16_t i ; 54 | for (i = 0; i < BURST_SIZE; i++) 55 | { 56 | eth_hdr = rte_pktmbuf_mtod(pkts[i], struct rte_ether_hdr *); 57 | eth_hdr->d_addr = d_addr; 58 | eth_hdr->s_addr = s_addr; 59 | eth_hdr->ether_type = ether_type; 60 | 61 | ipv4_hdr = rte_pktmbuf_mtod_offset(pkts[i], struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr)); 62 | ipv4_hdr->version_ihl = 0x45; 63 | ipv4_hdr->next_proto_id = 0x11; 64 | ipv4_hdr->src_addr = s_ip_addr; 65 | ipv4_hdr->dst_addr = d_ip_addr; 66 | ipv4_hdr->time_to_live = 0x40; 67 | 68 | udp_hdr = rte_pktmbuf_mtod_offset(pkts[i], struct rte_udp_hdr *, sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr)); 69 | udp_hdr->dgram_len = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr)); 70 | udp_hdr->src_port = rte_cpu_to_be_16(1234+i); 71 | udp_hdr->dst_port = rte_cpu_to_be_16(6666); 72 | ipv4_hdr->total_length = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr) + sizeof(struct rte_ipv4_hdr)); 73 | 74 | //init udp payload 75 | struct SRoU obj = { 76 | .magic_num = 1, 77 | .srou_length = 4, 78 | .flags = 0xFF, 79 | .next_protcol = 0, 80 | }; 81 | struct SRoU *msg; 82 | 83 | msg = (struct SRoU *)(rte_pktmbuf_mtod(pkts[i], char *) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr)); 84 | *msg = obj; 85 | 86 | int pkt_size = sizeof(struct SRoU) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr); 87 | 88 | pkts[i]->l2_len = sizeof(struct rte_ether_hdr); 89 | pkts[i]->l3_len = sizeof(struct rte_ipv4_hdr); 90 | pkts[i]->l4_len = sizeof(struct rte_udp_hdr); 91 | pkts[i]->ol_flags |= PKT_TX_IPV4 | PKT_TX_IP_CKSUM | PKT_TX_UDP_CKSUM; 92 | ipv4_hdr->hdr_checksum = 0; 93 | udp_hdr->dgram_cksum = rte_ipv4_phdr_cksum(ipv4_hdr, pkts[i]->ol_flags); 94 | pkts[i]->data_len = pkt_size; 95 | pkts[i]->pkt_len = pkt_size; 96 | 97 | } 98 | 99 | uint16_t sent = rte_eth_tx_burst(0, p->tx_queue_id, pkts, BURST_SIZE); 100 | if (unlikely(sent < BURST_SIZE)) 101 | { 102 | while (sent < BURST_SIZE) 103 | { 104 | rte_pktmbuf_free(pkts[sent++]); 105 | } 106 | } 107 | 108 | } 109 | 110 | return 0; 111 | } 112 | -------------------------------------------------------------------------------- /03_ring/02_pktRing/send_pkt.c: -------------------------------------------------------------------------------- 1 | 2 | #include "send_pkt.h" 3 | 4 | rte_be32_t string_to_ip(char *s) 5 | { 6 | unsigned char a[4]; 7 | int rc = sscanf(s, "%hhd.%hhd.%hhd.%hhd", a + 0, a + 1, a + 2, a + 3); 8 | if (rc != 4) 9 | { 10 | fprintf(stderr, "bad source IP address format. Use like: 1.1.1.1\n"); 11 | exit(1); 12 | } 13 | return (rte_be32_t)(a[3]) << 24 | 14 | (rte_be32_t)(a[2]) << 16 | 15 | (rte_be32_t)(a[1]) << 8 | 16 | (rte_be32_t)(a[0]); 17 | } 18 | 19 | int lcore_send_pkt(struct lcore_params *p) 20 | { 21 | const int socket_id = rte_socket_id(); 22 | printf("\nCore %u doing packet enqueue.\n", rte_lcore_id()); 23 | 24 | struct rte_ether_hdr *eth_hdr; 25 | struct rte_ipv4_hdr *ipv4_hdr; 26 | struct rte_udp_hdr *udp_hdr; 27 | 28 | //init mac 29 | struct rte_ether_addr s_addr = {{0xec, 0x0d, 0x9a, 0xc5, 0xdf, 0xfc}}; 30 | struct rte_ether_addr d_addr = {{0xec, 0x0d, 0x9a, 0xc5, 0xdf, 0xfd}}; 31 | 32 | //init IP header 33 | rte_be32_t s_ip_addr = string_to_ip("1.0.0.253"); 34 | rte_be32_t d_ip_addr = string_to_ip("1.0.0.1"); 35 | uint16_t ether_type = rte_cpu_to_be_16(0x0800); 36 | //Defined header in UDP 37 | struct SRoU 38 | { 39 | uint8_t magic_num; 40 | uint8_t srou_length; 41 | uint8_t flags; 42 | uint8_t next_protcol; 43 | uint64_t pad; 44 | }; 45 | 46 | struct rte_mbuf *pkts[BURST_SIZE]; 47 | 48 | for (;;) 49 | { 50 | rte_pktmbuf_alloc_bulk(p->mem_pool, pkts, BURST_SIZE); 51 | 52 | for (int i = 0; i < BURST_SIZE; i++) 53 | { 54 | //pkts[i] = rte_pktmbuf_alloc(p->mem_pool); 55 | //build_pkt(pkts[i]); 56 | 57 | eth_hdr = rte_pktmbuf_mtod(pkts[i], struct rte_ether_hdr *); 58 | eth_hdr->d_addr = d_addr; 59 | eth_hdr->s_addr = s_addr; 60 | eth_hdr->ether_type = ether_type; 61 | 62 | ipv4_hdr = rte_pktmbuf_mtod_offset(pkts[i], struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr)); 63 | ipv4_hdr->version_ihl = 0x45; 64 | ipv4_hdr->next_proto_id = 0x11; 65 | ipv4_hdr->src_addr = s_ip_addr; 66 | ipv4_hdr->dst_addr = d_ip_addr; 67 | ipv4_hdr->time_to_live = 0x40; 68 | 69 | udp_hdr = rte_pktmbuf_mtod_offset(pkts[i], struct rte_udp_hdr *, sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr)); 70 | udp_hdr->dgram_len = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr)); 71 | udp_hdr->src_port = rte_cpu_to_be_16(1234); 72 | udp_hdr->dst_port = rte_cpu_to_be_16(6666); 73 | ipv4_hdr->total_length = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr) + sizeof(struct rte_ipv4_hdr)); 74 | 75 | //init udp payload 76 | struct SRoU obj = { 77 | .magic_num = 1, 78 | .srou_length = 4, 79 | .flags = 0xFF, 80 | .next_protcol = 0, 81 | }; 82 | struct SRoU *msg; 83 | 84 | msg = (struct SRoU *)(rte_pktmbuf_mtod(pkts[i], char *) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr)); 85 | *msg = obj; 86 | 87 | int pkt_size = sizeof(struct SRoU) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr); 88 | 89 | pkts[i]->l2_len = sizeof(struct rte_ether_hdr); 90 | pkts[i]->l3_len = sizeof(struct rte_ipv4_hdr); 91 | pkts[i]->l4_len = sizeof(struct rte_udp_hdr); 92 | pkts[i]->ol_flags |= PKT_TX_IPV4 | PKT_TX_IP_CKSUM | PKT_TX_UDP_CKSUM; 93 | ipv4_hdr->hdr_checksum = 0; 94 | udp_hdr->dgram_cksum = rte_ipv4_phdr_cksum(ipv4_hdr, pkts[i]->ol_flags); 95 | pkts[i]->data_len = pkt_size; 96 | pkts[i]->pkt_len = pkt_size; 97 | } 98 | 99 | uint16_t nb_send = BURST_SIZE; 100 | uint16_t sent = rte_ring_sp_enqueue_bulk(p->ring, (void *)pkts, nb_send, NULL); 101 | 102 | if (unlikely(sent < nb_send)) 103 | { 104 | while (sent < nb_send) 105 | { 106 | rte_pktmbuf_free(pkts[sent++]); 107 | } 108 | } 109 | } 110 | return 0; 111 | } 112 | -------------------------------------------------------------------------------- /04_rss/multiThreadRing/send_pkt.c: -------------------------------------------------------------------------------- 1 | 2 | #include "send_pkt.h" 3 | 4 | rte_be32_t string_to_ip(char *s) 5 | { 6 | unsigned char a[4]; 7 | int rc = sscanf(s, "%hhd.%hhd.%hhd.%hhd", a + 0, a + 1, a + 2, a + 3); 8 | if (rc != 4) 9 | { 10 | fprintf(stderr, "bad source IP address format. Use like: 1.1.1.1\n"); 11 | exit(1); 12 | } 13 | return (rte_be32_t)(a[3]) << 24 | 14 | (rte_be32_t)(a[2]) << 16 | 15 | (rte_be32_t)(a[1]) << 8 | 16 | (rte_be32_t)(a[0]); 17 | } 18 | 19 | 20 | int lcore_send_pkt(struct lcore_params *p) 21 | { 22 | const int socket_id = rte_socket_id(); 23 | printf("Core %u doing packet enqueue.\n", rte_lcore_id()); 24 | 25 | struct rte_ether_hdr *eth_hdr; 26 | struct rte_ipv4_hdr *ipv4_hdr; 27 | struct rte_udp_hdr *udp_hdr; 28 | 29 | //init mac 30 | struct rte_ether_addr s_addr = {{0xec, 0x0d, 0x9a, 0xc5, 0xdf, 0xfc}}; 31 | struct rte_ether_addr d_addr = {{0xec, 0x0d, 0x9a, 0xc5, 0xdf, 0xfd}}; 32 | 33 | //init IP header 34 | rte_be32_t s_ip_addr = string_to_ip("1.0.0.253"); 35 | rte_be32_t d_ip_addr = string_to_ip("1.0.0.1"); 36 | uint16_t ether_type = rte_cpu_to_be_16(0x0800); 37 | //Defined header in UDP 38 | struct SRoU 39 | { 40 | uint8_t magic_num; 41 | uint8_t srou_length; 42 | uint8_t flags; 43 | uint8_t next_protcol; 44 | uint64_t pad; 45 | }; 46 | 47 | struct rte_mbuf *pkts[BURST_SIZE]; 48 | 49 | for (;;) 50 | { 51 | rte_pktmbuf_alloc_bulk(p->mem_pool, pkts, BURST_SIZE); 52 | 53 | for (int i = 0; i < BURST_SIZE; i++) 54 | { 55 | //pkts[i] = rte_pktmbuf_alloc(p->mem_pool); 56 | //build_pkt(pkts[i]); 57 | 58 | eth_hdr = rte_pktmbuf_mtod(pkts[i], struct rte_ether_hdr *); 59 | eth_hdr->d_addr = d_addr; 60 | eth_hdr->s_addr = s_addr; 61 | eth_hdr->ether_type = ether_type; 62 | 63 | ipv4_hdr = rte_pktmbuf_mtod_offset(pkts[i], struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr)); 64 | ipv4_hdr->version_ihl = 0x45; 65 | ipv4_hdr->next_proto_id = 0x11; 66 | ipv4_hdr->src_addr = s_ip_addr; 67 | ipv4_hdr->dst_addr = d_ip_addr; 68 | ipv4_hdr->time_to_live = 0x40; 69 | 70 | udp_hdr = rte_pktmbuf_mtod_offset(pkts[i], struct rte_udp_hdr *, sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr)); 71 | udp_hdr->dgram_len = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr)); 72 | udp_hdr->src_port = rte_cpu_to_be_16(1234+i); 73 | udp_hdr->dst_port = rte_cpu_to_be_16(6666); 74 | ipv4_hdr->total_length = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr) + sizeof(struct rte_ipv4_hdr)); 75 | 76 | //init udp payload 77 | struct SRoU obj = { 78 | .magic_num = 1, 79 | .srou_length = 4, 80 | .flags = 0xFF, 81 | .next_protcol = 0, 82 | }; 83 | struct SRoU *msg; 84 | 85 | msg = (struct SRoU *)(rte_pktmbuf_mtod(pkts[i], char *) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr)); 86 | *msg = obj; 87 | 88 | int pkt_size = sizeof(struct SRoU) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr); 89 | 90 | pkts[i]->l2_len = sizeof(struct rte_ether_hdr); 91 | pkts[i]->l3_len = sizeof(struct rte_ipv4_hdr); 92 | pkts[i]->l4_len = sizeof(struct rte_udp_hdr); 93 | pkts[i]->ol_flags |= PKT_TX_IPV4 | PKT_TX_IP_CKSUM | PKT_TX_UDP_CKSUM; 94 | ipv4_hdr->hdr_checksum = 0; 95 | udp_hdr->dgram_cksum = rte_ipv4_phdr_cksum(ipv4_hdr, pkts[i]->ol_flags); 96 | pkts[i]->data_len = pkt_size; 97 | pkts[i]->pkt_len = pkt_size; 98 | } 99 | 100 | uint16_t nb_send = BURST_SIZE; 101 | uint16_t sent = rte_ring_sp_enqueue_bulk(p->tx_ring, (void *)pkts, nb_send, NULL); 102 | 103 | //unsigned int sent = rte_eth_tx_burst(0, p->worker_id, pkts, nb_send); 104 | if (unlikely(sent < nb_send)) 105 | { 106 | while (sent < nb_send) 107 | { 108 | rte_pktmbuf_free(pkts[sent++]); 109 | } 110 | } 111 | } 112 | return 0; 113 | } 114 | -------------------------------------------------------------------------------- /03_ring/03_singleThreadPktRing/send_pkt.c: -------------------------------------------------------------------------------- 1 | 2 | #include "send_pkt.h" 3 | 4 | rte_be32_t string_to_ip(char *s) 5 | { 6 | unsigned char a[4]; 7 | int rc = sscanf(s, "%hhd.%hhd.%hhd.%hhd", a + 0, a + 1, a + 2, a + 3); 8 | if (rc != 4) 9 | { 10 | fprintf(stderr, "bad source IP address format. Use like: 1.1.1.1\n"); 11 | exit(1); 12 | } 13 | return (rte_be32_t)(a[3]) << 24 | 14 | (rte_be32_t)(a[2]) << 16 | 15 | (rte_be32_t)(a[1]) << 8 | 16 | (rte_be32_t)(a[0]); 17 | } 18 | 19 | 20 | int lcore_send_pkt(struct lcore_params *p) 21 | { 22 | const int socket_id = rte_socket_id(); 23 | printf("\nCore %u doing packet enqueue.\n", rte_lcore_id()); 24 | 25 | struct rte_ether_hdr *eth_hdr; 26 | struct rte_ipv4_hdr *ipv4_hdr; 27 | struct rte_udp_hdr *udp_hdr; 28 | 29 | //init mac 30 | struct rte_ether_addr s_addr = {{0xec, 0x0d, 0x9a, 0xc5, 0xdf, 0xfc}}; 31 | struct rte_ether_addr d_addr = {{0xec, 0x0d, 0x9a, 0xc5, 0xdf, 0xfd}}; 32 | 33 | //init IP header 34 | rte_be32_t s_ip_addr = string_to_ip("1.0.0.253"); 35 | rte_be32_t d_ip_addr = string_to_ip("1.0.0.1"); 36 | uint16_t ether_type = rte_cpu_to_be_16(0x0800); 37 | //Defined header in UDP 38 | struct SRoU 39 | { 40 | uint8_t magic_num; 41 | uint8_t srou_length; 42 | uint8_t flags; 43 | uint8_t next_protcol; 44 | uint64_t pad; 45 | }; 46 | 47 | struct rte_mbuf *pkts[BURST_SIZE]; 48 | 49 | for (;;) 50 | { 51 | rte_pktmbuf_alloc_bulk(p->mem_pool, pkts, BURST_SIZE); 52 | 53 | for (int i = 0; i < BURST_SIZE; i++) 54 | { 55 | //pkts[i] = rte_pktmbuf_alloc(p->mem_pool); 56 | //build_pkt(pkts[i]); 57 | 58 | eth_hdr = rte_pktmbuf_mtod(pkts[i], struct rte_ether_hdr *); 59 | eth_hdr->d_addr = d_addr; 60 | eth_hdr->s_addr = s_addr; 61 | eth_hdr->ether_type = ether_type; 62 | 63 | ipv4_hdr = rte_pktmbuf_mtod_offset(pkts[i], struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr)); 64 | ipv4_hdr->version_ihl = 0x45; 65 | ipv4_hdr->next_proto_id = 0x11; 66 | ipv4_hdr->src_addr = s_ip_addr; 67 | ipv4_hdr->dst_addr = d_ip_addr; 68 | ipv4_hdr->time_to_live = 0x40; 69 | 70 | udp_hdr = rte_pktmbuf_mtod_offset(pkts[i], struct rte_udp_hdr *, sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr)); 71 | udp_hdr->dgram_len = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr)); 72 | udp_hdr->src_port = rte_cpu_to_be_16(1234); 73 | udp_hdr->dst_port = rte_cpu_to_be_16(6666); 74 | ipv4_hdr->total_length = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr) + sizeof(struct rte_ipv4_hdr)); 75 | 76 | //init udp payload 77 | struct SRoU obj = { 78 | .magic_num = 1, 79 | .srou_length = 4, 80 | .flags = 0xFF, 81 | .next_protcol = 0, 82 | }; 83 | struct SRoU *msg; 84 | 85 | msg = (struct SRoU *)(rte_pktmbuf_mtod(pkts[i], char *) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr)); 86 | *msg = obj; 87 | 88 | int pkt_size = sizeof(struct SRoU) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr); 89 | 90 | pkts[i]->l2_len = sizeof(struct rte_ether_hdr); 91 | pkts[i]->l3_len = sizeof(struct rte_ipv4_hdr); 92 | pkts[i]->l4_len = sizeof(struct rte_udp_hdr); 93 | pkts[i]->ol_flags |= PKT_TX_IPV4 | PKT_TX_IP_CKSUM | PKT_TX_UDP_CKSUM; 94 | ipv4_hdr->hdr_checksum = 0; 95 | udp_hdr->dgram_cksum = rte_ipv4_phdr_cksum(ipv4_hdr, pkts[i]->ol_flags); 96 | pkts[i]->data_len = pkt_size; 97 | pkts[i]->pkt_len = pkt_size; 98 | } 99 | 100 | uint16_t nb_send = BURST_SIZE; 101 | uint16_t sent = rte_ring_sp_enqueue_bulk(p->tx_ring, (void *)pkts, nb_send, NULL); 102 | 103 | //unsigned int sent = rte_eth_tx_burst(0, p->worker_id, pkts, nb_send); 104 | if (unlikely(sent < nb_send)) 105 | { 106 | while (sent < nb_send) 107 | { 108 | rte_pktmbuf_free(pkts[sent++]); 109 | } 110 | } 111 | } 112 | return 0; 113 | } 114 | -------------------------------------------------------------------------------- /02_send_recv/traffic_gen/portinit.c: -------------------------------------------------------------------------------- 1 | #include "portinit.h" 2 | 3 | int port_init(uint16_t port, struct rte_mempool *mbuf_pool) 4 | { 5 | struct rte_eth_conf port_conf = port_conf_default; 6 | const uint16_t rx_rings = NUM_RX_QUEUE, tx_rings = NUM_TX_QUEUE; 7 | uint16_t nb_rxd = RX_RING_SIZE; 8 | uint16_t nb_txd = TX_RING_SIZE; 9 | int retval; 10 | uint16_t q; 11 | struct rte_eth_dev_info dev_info; 12 | struct rte_eth_txconf txconf; 13 | 14 | if (!rte_eth_dev_is_valid_port(port)) 15 | return -1; 16 | 17 | retval = rte_eth_dev_info_get(port, &dev_info); 18 | if (retval != 0) 19 | { 20 | printf("Error during getting device (port %u) info: %s\n", 21 | port, strerror(-retval)); 22 | return retval; 23 | } 24 | printf("\n\ninitializing port %d...\n", port); 25 | 26 | if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_CHECKSUM) 27 | { 28 | printf("port[%u] support RX cheksum offload.\n", port); 29 | port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CHECKSUM; 30 | } 31 | 32 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 33 | { 34 | printf("port[%u] support TX mbuf fast free offload.\n", port); 35 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE; 36 | } 37 | 38 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MT_LOCKFREE) 39 | { 40 | printf("port[%u] support TX MT lock free offload.\n", port); 41 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MT_LOCKFREE; 42 | } 43 | 44 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 45 | { 46 | printf("port[%u] support TX IPv4 checksum offload.\n", port); 47 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 48 | } 49 | 50 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) 51 | { 52 | printf("port[%u] support TX UDP checksum offload.\n", port); 53 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_UDP_CKSUM; 54 | } 55 | 56 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM) 57 | { 58 | printf("port[%u] support TX TCP checksum offload.\n", port); 59 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_TCP_CKSUM; 60 | } 61 | 62 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM) 63 | { 64 | printf("port[%u] support TX SCTP checksum offload.\n", port); 65 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_SCTP_CKSUM; 66 | } 67 | 68 | /* Configure the Ethernet device. */ 69 | retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); 70 | if (retval != 0) 71 | return retval; 72 | 73 | retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); 74 | if (retval != 0) 75 | return retval; 76 | 77 | /* Allocate and set up multiple RX queue per Ethernet port. */ 78 | for (q = 0; q < rx_rings; q++) 79 | { 80 | retval = rte_eth_rx_queue_setup(port, q, nb_rxd, 81 | rte_eth_dev_socket_id(port), NULL, mbuf_pool); 82 | if (retval < 0) 83 | return retval; 84 | } 85 | 86 | txconf = dev_info.default_txconf; 87 | txconf.offloads = port_conf.txmode.offloads; 88 | /* Allocate and set up multiple TX queue per Ethernet port. */ 89 | for (q = 0; q < tx_rings; q++) 90 | { 91 | retval = rte_eth_tx_queue_setup(port, q, nb_txd, 92 | rte_eth_dev_socket_id(port), &txconf); 93 | if (retval < 0) 94 | return retval; 95 | } 96 | 97 | /* Start the Ethernet port. */ 98 | retval = rte_eth_dev_start(port); 99 | if (retval < 0) 100 | return retval; 101 | 102 | struct rte_eth_link link; 103 | do 104 | { 105 | retval = rte_eth_link_get_nowait(port, &link); 106 | if (retval < 0) 107 | { 108 | printf("Failed link get (port %u): %s\n", 109 | port, rte_strerror(-retval)); 110 | return retval; 111 | } 112 | else if (link.link_status) 113 | break; 114 | 115 | printf("Waiting for Link up on port %" PRIu16 "\n", port); 116 | sleep(1); 117 | } while (!link.link_status); 118 | 119 | /* Display the port MAC address. */ 120 | struct rte_ether_addr addr; 121 | retval = rte_eth_macaddr_get(port, &addr); 122 | if (retval != 0) 123 | return retval; 124 | 125 | printf("Port[%u] MAC: %02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 126 | ":%02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 "\n", 127 | port, 128 | addr.addr_bytes[0], addr.addr_bytes[1], 129 | addr.addr_bytes[2], addr.addr_bytes[3], 130 | addr.addr_bytes[4], addr.addr_bytes[5]); 131 | 132 | /* Enable RX in promiscuous mode for the Ethernet device. */ 133 | retval = rte_eth_promiscuous_enable(port); 134 | if (retval != 0) 135 | return retval; 136 | 137 | return 0; 138 | } 139 | -------------------------------------------------------------------------------- /03_ring/03_singleThreadPktRing/portinit.c: -------------------------------------------------------------------------------- 1 | #include "portinit.h" 2 | 3 | static const struct rte_eth_conf port_conf_default = { 4 | .rxmode = { 5 | .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 6 | }, 7 | .txmode = { 8 | .mq_mode = ETH_MQ_TX_NONE, 9 | } 10 | 11 | }; 12 | 13 | int port_init(uint16_t port, struct rte_mempool *mbuf_pool) 14 | { 15 | struct rte_eth_conf port_conf = port_conf_default; 16 | const uint16_t rx_rings = NUM_RX_QUEUE, tx_rings = NUM_TX_QUEUE; 17 | uint16_t nb_rxd = RX_RING_SIZE; 18 | uint16_t nb_txd = TX_RING_SIZE; 19 | int retval; 20 | uint16_t q; 21 | struct rte_eth_dev_info dev_info; 22 | struct rte_eth_txconf txconf; 23 | 24 | if (!rte_eth_dev_is_valid_port(port)) 25 | return -1; 26 | 27 | retval = rte_eth_dev_info_get(port, &dev_info); 28 | if (retval != 0) 29 | { 30 | printf("Error during getting device (port %u) info: %s\n", 31 | port, strerror(-retval)); 32 | return retval; 33 | } 34 | printf("\n\ninitializing port %d...\n", port); 35 | 36 | if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_CHECKSUM) 37 | { 38 | printf("port[%u] support RX cheksum offload.\n", port); 39 | port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CHECKSUM; 40 | } 41 | 42 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 43 | { 44 | printf("port[%u] support TX mbuf fast free offload.\n", port); 45 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE; 46 | } 47 | 48 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MT_LOCKFREE) 49 | { 50 | printf("port[%u] support TX MT lock free offload.\n", port); 51 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MT_LOCKFREE; 52 | } 53 | 54 | 55 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 56 | { 57 | printf("port[%u] support TX IPv4 checksum offload.\n", port); 58 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 59 | } 60 | 61 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) 62 | { 63 | printf("port[%u] support TX UDP checksum offload.\n", port); 64 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_UDP_CKSUM; 65 | } 66 | 67 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM) 68 | { 69 | printf("port[%u] support TX TCP checksum offload.\n", port); 70 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_TCP_CKSUM; 71 | } 72 | 73 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM) 74 | { 75 | printf("port[%u] support TX SCTP checksum offload.\n", port); 76 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_SCTP_CKSUM; 77 | } 78 | 79 | /* Configure the Ethernet device. */ 80 | retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); 81 | if (retval != 0) 82 | return retval; 83 | 84 | retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); 85 | if (retval != 0) 86 | return retval; 87 | 88 | /* Allocate and set up 1 RX queue per Ethernet port. */ 89 | for (q = 0; q < rx_rings; q++) 90 | { 91 | retval = rte_eth_rx_queue_setup(port, q, nb_rxd, 92 | rte_eth_dev_socket_id(port), NULL, mbuf_pool); 93 | if (retval < 0) 94 | return retval; 95 | } 96 | 97 | txconf = dev_info.default_txconf; 98 | txconf.offloads = port_conf.txmode.offloads; 99 | /* Allocate and set up 1 TX queue per Ethernet port. */ 100 | for (q = 0; q < tx_rings; q++) 101 | { 102 | retval = rte_eth_tx_queue_setup(port, q, nb_txd, 103 | rte_eth_dev_socket_id(port), &txconf); 104 | if (retval < 0) 105 | return retval; 106 | } 107 | 108 | /* Start the Ethernet port. */ 109 | retval = rte_eth_dev_start(port); 110 | if (retval < 0) 111 | return retval; 112 | 113 | struct rte_eth_link link; 114 | do 115 | { 116 | retval = rte_eth_link_get_nowait(port, &link); 117 | if (retval < 0) 118 | { 119 | printf("Failed link get (port %u): %s\n", 120 | port, rte_strerror(-retval)); 121 | return retval; 122 | } 123 | else if (link.link_status) 124 | break; 125 | 126 | printf("Waiting for Link up on port %" PRIu16 "\n", port); 127 | sleep(1); 128 | } while (!link.link_status); 129 | 130 | /* Display the port MAC address. */ 131 | struct rte_ether_addr addr; 132 | retval = rte_eth_macaddr_get(port, &addr); 133 | if (retval != 0) 134 | return retval; 135 | 136 | printf("Port[%u] MAC: %02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 137 | ":%02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 "\n", 138 | port, 139 | addr.addr_bytes[0], addr.addr_bytes[1], 140 | addr.addr_bytes[2], addr.addr_bytes[3], 141 | addr.addr_bytes[4], addr.addr_bytes[5]); 142 | 143 | /* Enable RX in promiscuous mode for the Ethernet device. */ 144 | retval = rte_eth_promiscuous_enable(port); 145 | if (retval != 0) 146 | return retval; 147 | 148 | return 0; 149 | } -------------------------------------------------------------------------------- /04_rss/multiThreadRing/portinit.c: -------------------------------------------------------------------------------- 1 | #include "portinit.h" 2 | 3 | static const struct rte_eth_conf port_conf_default = { 4 | .rxmode = { 5 | .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 6 | .mq_mode = ETH_MQ_RX_RSS, 7 | }, 8 | .rx_adv_conf = { 9 | .rss_conf = { 10 | .rss_key = NULL, 11 | //mlx5 support hf: 12 | .rss_hf = ETH_RSS_IP | ETH_RSS_TCP | ETH_RSS_UDP, 13 | //i40e support hf: 14 | //.rss_hf = ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV4_TCP, 15 | }, 16 | }, 17 | .txmode = { 18 | .mq_mode = ETH_MQ_TX_NONE, 19 | } 20 | 21 | }; 22 | 23 | int port_init(uint16_t port, struct rte_mempool *mbuf_pool) 24 | { 25 | struct rte_eth_conf port_conf = port_conf_default; 26 | const uint16_t rx_rings = NUM_RX_QUEUE, tx_rings = NUM_TX_QUEUE; 27 | uint16_t nb_rxd = RX_RING_SIZE; 28 | uint16_t nb_txd = TX_RING_SIZE; 29 | int retval; 30 | uint16_t q; 31 | struct rte_eth_dev_info dev_info; 32 | struct rte_eth_txconf txconf; 33 | 34 | if (!rte_eth_dev_is_valid_port(port)) 35 | return -1; 36 | 37 | retval = rte_eth_dev_info_get(port, &dev_info); 38 | if (retval != 0) 39 | { 40 | printf("Error during getting device (port %u) info: %s\n", 41 | port, strerror(-retval)); 42 | return retval; 43 | } 44 | printf("\n\ninitializing port %d...\n", port); 45 | 46 | if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_CHECKSUM) 47 | { 48 | printf("port[%u] support RX cheksum offload.\n", port); 49 | port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CHECKSUM; 50 | } 51 | 52 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 53 | { 54 | printf("port[%u] support TX mbuf fast free offload.\n", port); 55 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE; 56 | } 57 | 58 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MT_LOCKFREE) 59 | { 60 | printf("port[%u] support TX MT lock free offload.\n", port); 61 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MT_LOCKFREE; 62 | } 63 | 64 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 65 | { 66 | printf("port[%u] support TX IPv4 checksum offload.\n", port); 67 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 68 | } 69 | 70 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) 71 | { 72 | printf("port[%u] support TX UDP checksum offload.\n", port); 73 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_UDP_CKSUM; 74 | } 75 | 76 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM) 77 | { 78 | printf("port[%u] support TX TCP checksum offload.\n", port); 79 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_TCP_CKSUM; 80 | } 81 | 82 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM) 83 | { 84 | printf("port[%u] support TX SCTP checksum offload.\n", port); 85 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_SCTP_CKSUM; 86 | } 87 | 88 | /* Configure the Ethernet device. */ 89 | retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); 90 | if (retval != 0) 91 | return retval; 92 | 93 | retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); 94 | if (retval != 0) 95 | return retval; 96 | 97 | /* Allocate and set up multiple RX queue per Ethernet port. */ 98 | for (q = 0; q < rx_rings; q++) 99 | { 100 | retval = rte_eth_rx_queue_setup(port, q, nb_rxd, 101 | rte_eth_dev_socket_id(port), NULL, mbuf_pool); 102 | if (retval < 0) 103 | return retval; 104 | } 105 | 106 | txconf = dev_info.default_txconf; 107 | txconf.offloads = port_conf.txmode.offloads; 108 | /* Allocate and set up multiple TX queue per Ethernet port. */ 109 | for (q = 0; q < tx_rings; q++) 110 | { 111 | retval = rte_eth_tx_queue_setup(port, q, nb_txd, 112 | rte_eth_dev_socket_id(port), &txconf); 113 | if (retval < 0) 114 | return retval; 115 | } 116 | 117 | /* Start the Ethernet port. */ 118 | retval = rte_eth_dev_start(port); 119 | if (retval < 0) 120 | return retval; 121 | 122 | struct rte_eth_link link; 123 | do 124 | { 125 | retval = rte_eth_link_get_nowait(port, &link); 126 | if (retval < 0) 127 | { 128 | printf("Failed link get (port %u): %s\n", 129 | port, rte_strerror(-retval)); 130 | return retval; 131 | } 132 | else if (link.link_status) 133 | break; 134 | 135 | printf("Waiting for Link up on port %" PRIu16 "\n", port); 136 | sleep(1); 137 | } while (!link.link_status); 138 | 139 | /* Display the port MAC address. */ 140 | struct rte_ether_addr addr; 141 | retval = rte_eth_macaddr_get(port, &addr); 142 | if (retval != 0) 143 | return retval; 144 | 145 | printf("Port[%u] MAC: %02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 146 | ":%02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 "\n", 147 | port, 148 | addr.addr_bytes[0], addr.addr_bytes[1], 149 | addr.addr_bytes[2], addr.addr_bytes[3], 150 | addr.addr_bytes[4], addr.addr_bytes[5]); 151 | 152 | /* Enable RX in promiscuous mode for the Ethernet device. */ 153 | retval = rte_eth_promiscuous_enable(port); 154 | if (retval != 0) 155 | return retval; 156 | 157 | return 0; 158 | } 159 | -------------------------------------------------------------------------------- /a1_setup_mlx5_sriov_env/ubuntu.md: -------------------------------------------------------------------------------- 1 | ## 安装和编译环境配置 2 | 3 | 安装ubuntu 20.04.2，然后安装的包如下,基本的编译环境，外加了我比较喜欢用terminator和tmux 4 | ```bash 5 | sudo apt update 6 | sudo apt upgrade 7 | 8 | sudo apt install openssh-server terminator tmux build-essential libnuma-dev 9 | ``` 10 | 11 | 然后修改grub，打开iommu 12 | ```bash 13 | sudo vi /etc/default/grub 14 | 15 | //原来 16 | GRUB_CMDLINE_LINUX_DEFAULT="quiet splash" 17 | //修改后 18 | GRUB_CMDLINE_LINUX_DEFAULT="quiet splash iommu=pt intel_iommu=on" 19 | ``` 20 | 21 | 最后更新grub，然后重启 22 | 23 | ```bash 24 | sudo grub-mkconfig -o /boot/grub/grub.cfg 25 | sudo reboot 26 | ``` 27 | 28 | 29 | ## 编译dpdk 30 | 31 | 下载dpdk并解压 32 | ```bash 33 | wget http://fast.dpdk.org/rel/dpdk-21.05.tar.xz 34 | tar xf dpdk-21.05.tar.xz 35 | cd dpdk-21.05 36 | ``` 37 | 安装meson和pyelftools 38 | ```bash 39 | sudo apt install meson 40 | sudo apt install python3-pyelftools 41 | ``` 42 | 43 | 编译和安装 44 | ```bash 45 | meson -Dexamples=all build 46 | 47 | cd build 48 | ninja 49 | sudo ninja install 50 | sudo ldconfig 51 | ``` 52 | 配置hugepages和绑定vfio 53 | ```bash 54 | sudo modprobe vfio-pci 55 | sudo dpdk-hugepages.py --setup 2G 56 | sudo dpdk-devbind.py --status 57 | 58 | Network devices using kernel driver 59 | =================================== 60 | 0000:03:00.0 'VMXNET3 Ethernet Controller 07b0' if=ens160 drv=vmxnet3 unused=vfio-pci *Active* 61 | 0000:04:00.0 'VMXNET3 Ethernet Controller 07b0' if=ens161 drv=vmxnet3 unused=vfio-pci 62 | 0000:0b:00.0 'VMXNET3 Ethernet Controller 07b0' if=ens192 drv=vmxnet3 unused=vfio-pci 63 | 0000:13:00.0 'VMXNET3 Ethernet Controller 07b0' if=ens224 drv=vmxnet3 unused=vfio-pci 64 | 0000:1b:00.0 'VMXNET3 Ethernet Controller 07b0' if=ens256 drv=vmxnet3 unused=vfio-pci 65 | 66 | sudo dpdk-devbind.py -b vfio-pci ens161 67 | sudo dpdk-devbind.py -b vfio-pci ens192 68 | sudo dpdk-devbind.py -b vfio-pci ens224 69 | sudo dpdk-devbind.py -b vfio-pci ens256 70 | ``` 71 | 72 | 73 | 74 | ## MLX5安装 75 | ```bash 76 | tar vzxf MLNX_OFED_LINUX-5.3-1.0.0.1-ubuntu20.04-x86_64.tgz 77 | cd MLNX_OFED_LINUX-5.3-1.0.0.1-ubuntu20.04-x86_64/ 78 | sudo ./mlnxofedinstall --upstream-libs --dpdk --add-kernel-support 79 | 80 | sudo /etc/init.d/openibd restart 81 | 82 | ``` 83 | 84 | 编译DPDK和前文一样 85 | ```bash 86 | zartbot@zartbot-KVM:~/dpdk-21.05/build$ sudo dpdk-devbind.py --status 87 | 88 | Network devices using kernel driver 89 | =================================== 90 | 0000:01:00.0 'Virtio network device 1041' if=enp1s0 drv=virtio-pci unused=vfio-pci *Active* 91 | 0000:06:00.0 'MT28800 Family [ConnectX-5 Ex Virtual Function] 101a' if=enp6s0 drv=mlx5_core unused=vfio-pci 92 | 0000:07:00.0 'MT28800 Family [ConnectX-5 Ex Virtual Function] 101a' if=enp7s0 drv=mlx5_core unused=vfio-pci 93 | 94 | No 'Baseband' devices detected 95 | ============================== 96 | 97 | No 'Crypto' devices detected 98 | ============================ 99 | 100 | No 'Eventdev' devices detected 101 | ============================== 102 | 103 | No 'Mempool' devices detected 104 | ============================= 105 | 106 | No 'Compress' devices detected 107 | ============================== 108 | 109 | No 'Misc (rawdev)' devices detected 110 | =================================== 111 | 112 | No 'Regex' devices detected 113 | =========================== 114 | ``` 115 | 同样支持MLX5_PMD不需要bind，直接就可以使用 116 | ```bash 117 | zartbot@zartbot-KVM:~/test$ make 118 | cc -O3 -include rte_config.h -march=native -I/usr/local/include -I/usr/include/libnl3 -DALLOW_EXPERIMENTAL_API main.c -o build/helloworld-shared -L/usr/local/lib/x86_64-linux-gnu -Wl,--as-needed -lrte_node -lrte_graph -lrte_bpf -lrte_flow_classify -lrte_pipeline -lrte_table -lrte_port -lrte_fib -lrte_ipsec -lrte_vhost -lrte_stack -lrte_security -lrte_sched -lrte_reorder -lrte_rib -lrte_regexdev -lrte_rawdev -lrte_pdump -lrte_power -lrte_member -lrte_lpm -lrte_latencystats -lrte_kni -lrte_jobstats -lrte_ip_frag -lrte_gso -lrte_gro -lrte_eventdev -lrte_efd -lrte_distributor -lrte_cryptodev -lrte_compressdev -lrte_cfgfile -lrte_bitratestats -lrte_bbdev -lrte_acl -lrte_timer -lrte_hash -lrte_metrics -lrte_cmdline -lrte_pci -lrte_ethdev -lrte_meter -lrte_net -lrte_mbuf -lrte_mempool -lrte_rcu -lrte_ring -lrte_eal -lrte_telemetry -lrte_kvargs 119 | ln -sf helloworld-shared build/helloworld 120 | zartbot@zartbot-KVM:~/test$ sudo ./build/helloworld 121 | EAL: Detected 4 lcore(s) 122 | EAL: Detected 1 NUMA nodes 123 | EAL: Detected shared linkage of DPDK 124 | EAL: Multi-process socket /var/run/dpdk/rte/mp_socket 125 | EAL: Selected IOVA mode 'PA' 126 | EAL: No available 1048576 kB hugepages reported 127 | EAL: VFIO support initialized 128 | EAL: Invalid NUMA socket, default to 0 129 | EAL: Probe PCI driver: net_virtio (1af4:1041) device: 0000:01:00.0 (socket 0) 130 | eth_virtio_pci_init(): Failed to init PCI device 131 | 132 | EAL: Requested device 0000:01:00.0 cannot be used 133 | EAL: Invalid NUMA socket, default to 0 134 | EAL: Probe PCI driver: mlx5_pci (15b3:101a) device: 0000:06:00.0 (socket 0) 135 | EAL: Invalid NUMA socket, default to 0 136 | EAL: Probe PCI driver: mlx5_pci (15b3:101a) device: 0000:07:00.0 (socket 0) 137 | TELEMETRY: No legacy callbacks, legacy socket not created 138 | 139 | 140 | 141 | ***************************************** 142 | number of available port: 2 143 | 144 | 145 | initializing port 0... 146 | port[0] support RX cheksum offload. 147 | port[0] support TX mbuf fast free offload. 148 | port[0] support TX IPv4 checksum offload. 149 | port[0] support TX UDP checksum offload. 150 | port[0] support TX TCP checksum offload. 151 | Port[0] MAC: 7a:99:ed:5f:e3:a6 152 | 153 | 154 | initializing port 1... 155 | port[1] support RX cheksum offload. 156 | port[1] support TX mbuf fast free offload. 157 | port[1] support TX IPv4 checksum offload. 158 | port[1] support TX UDP checksum offload. 159 | port[1] support TX TCP checksum offload. 160 | Port[1] MAC: 5a:d8:51:db:17:2d 161 | PPS: 2081708 162 | PPS: 23912180 163 | PPS: 18424908 164 | PPS: 8359624 165 | PPS: 9097548 166 | PPS: 18615392 167 | PPS: 8275396 168 | PPS: 8288376 169 | PPS: 8486400 170 | PPS: 9241732 171 | PPS: 18517960 172 | PPS: 8314496 173 | PPS: 8345988 174 | ``` 175 | 176 | -------------------------------------------------------------------------------- /02_send_recv/README.md: -------------------------------------------------------------------------------- 1 | ## 收包程序 2 | 收包主要是采用rte_eth_rx_burst函数，首先需要从mbuf_pool中分配一些空间,采用dpdk自带的rte_pktmbuf_alloc函数 3 | ```c 4 | struct rte_mbuf *rx_pkt[BURST_SIZE]; 5 | for (int i = 0; i < BURST_SIZE; i++) 6 | { 7 | rx_pkt[i] = rte_pktmbuf_alloc(mbuf_pool); 8 | } 9 | ``` 10 | 11 | 然后一个for循环，不停的接收就行了,接收时rte_eth_rx_burst(1, 0, rx_pkt, BURST_SIZE)中第一个参数为portid，第二个为队列id，由于我们这个示例每个接口只有一个队列，同时我们采用loopback cable把port0和port1对连的，因此主要就是一个port0发，port1收的场景,因此接收的portid=1. 收到报文后可以通过rte_pktbuf_mtod函数去解析报文，然后可以通过结构体内变量赋值的方式修改值，更具体的示例我们在发包函数里讲， 这里只有一个简单的parse源MAC的地址的场景 12 | ```c 13 | for (;;) 14 | { 15 | 16 | uint16_t nb_rx = rte_eth_rx_burst(1, 0, rx_pkt, BURST_SIZE); 17 | if (nb_rx == 0) 18 | { 19 | continue; 20 | } 21 | struct rte_ether_hdr *eth_hdr; 22 | 23 | for (int i = 0; i < nb_rx; i++) 24 | { 25 | eth_hdr = rte_pktmbuf_mtod(rx_pkt[i], struct rte_ether_hdr *); 26 | printf("Recv Pkt[%d] from MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 27 | " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " \n",i, 28 | eth_hdr->s_addr.addr_bytes[0], eth_hdr->s_addr.addr_bytes[1], 29 | eth_hdr->s_addr.addr_bytes[2], eth_hdr->s_addr.addr_bytes[3], 30 | eth_hdr->s_addr.addr_bytes[4], eth_hdr->s_addr.addr_bytes[5]); 31 | rte_pktmbuf_free(rx_pkt[i]); 32 | } 33 | } 34 | ``` 35 | ## 发包程序 36 | 本次发包程序的示例是以UDP发包为主，因此我们需要逐层初始化报文,然后报文的发送可以Burst的方式一次发送32个，我们也用这种方式来处理，报文发送的函数如下rte_eth_tx_burst. 37 | 38 | 首先我们初始化源目的MAC地址、IP地址和UDP Payload里面的内容，我们以SRoU header的一部分为例,相关的结构体定义如下： 39 | ```c 40 | struct rte_ether_hdr *eth_hdr; 41 | struct rte_ipv4_hdr *ipv4_hdr; 42 | struct rte_udp_hdr *udp_hdr; 43 | 44 | 45 | //Defined header in UDP 46 | struct SRoU 47 | { 48 | uint8_t magic_num; 49 | uint8_t srou_length; 50 | uint8_t flags; 51 | uint8_t next_protcol; 52 | uint64_t pad; 53 | }; 54 | ``` 55 | 然后我们分别来初始化每一层 56 | ```c 57 | //init mac 58 | struct rte_ether_addr s_addr = {{0x14, 0x02, 0xEC, 0x89, 0x8D, 0x24}}; 59 | struct rte_ether_addr d_addr = {{0x3c, 0xfd, 0xfe, 0xa9, 0xa8, 0x89}}; 60 | 61 | //init IP header 62 | rte_be32_t s_ip_addr = string_to_ip("1.0.0.253"); 63 | rte_be32_t d_ip_addr = string_to_ip("1.0.0.1"); 64 | uint16_t ether_type = rte_cpu_to_be_16(0x0800); 65 | 66 | //init udp payload 67 | struct SRoU obj = { 68 | .magic_num = 1, 69 | .srou_length = 4, 70 | .flags = 0xFF, 71 | .next_protcol = 0, 72 | }; 73 | ``` 74 | 初始化IP地址时有一个函数是从string转换为be32值 75 | ```c 76 | rte_be32_t string_to_ip(char *s) 77 | { 78 | unsigned char a[4]; 79 | int rc = sscanf(s, "%hhd.%hhd.%hhd.%hhd", a + 0, a + 1, a + 2, a + 3); 80 | if (rc != 4) 81 | { 82 | fprintf(stderr, "bad source IP address format. Use like: 1.2.3.4\n"); 83 | exit(1); 84 | } 85 | return (rte_be32_t)(a[3]) << 24 | 86 | (rte_be32_t)(a[2]) << 16 | 87 | (rte_be32_t)(a[1]) << 8 | 88 | (rte_be32_t)(a[0]); 89 | } 90 | ``` 91 | 接下来我们来从mbuf中分配空间并初始化每个报文,注意其中rte_pktmbuf_mtod_offset函数的用法，大量的报文修改都采用这种方式. 92 | ```c 93 | struct SRoU *msg; 94 | struct rte_mbuf *pkt[BURST_SIZE]; 95 | 96 | for (int i = 0; i < BURST_SIZE; i++) 97 | { 98 | //分配空间 99 | pkt[i] = rte_pktmbuf_alloc(mbuf_pool); 100 | 101 | //利用rte_pktmbuf_mtod函数修改二层头， 102 | eth_hdr = rte_pktmbuf_mtod(pkt[i], struct rte_ether_hdr *); 103 | eth_hdr->d_addr = d_addr; 104 | 105 | //这里我们根据burst循环改改源MAC地址玩~ 106 | struct rte_ether_addr s_addr = {{0x14, 0x02, 0xEC, 0x89, 0x8D, i}}; 107 | eth_hdr->s_addr = s_addr; 108 | eth_hdr->ether_type = ether_type; 109 | 110 | //然后利用rte_pktmbuf_mtod_offset函数， 移到IPv4头开始的地方，并定义结构体 111 | ipv4_hdr = rte_pktmbuf_mtod_offset(pkt[i], struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr)); 112 | ipv4_hdr->version_ihl = 0x45; 113 | ipv4_hdr->next_proto_id = 0x11; 114 | ipv4_hdr->src_addr = s_ip_addr; 115 | ipv4_hdr->dst_addr = d_ip_addr; 116 | ipv4_hdr->time_to_live = 0x40; 117 | 118 | 119 | //修改UDP头，注意大端小端转换的rte_cpu_to_be_16函数 120 | udp_hdr = rte_pktmbuf_mtod_offset(pkt[i], struct rte_udp_hdr *, sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr)); 121 | udp_hdr->dgram_len = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr)); 122 | udp_hdr->src_port = rte_cpu_to_be_16(1234); 123 | udp_hdr->dst_port = rte_cpu_to_be_16(6666); 124 | 125 | msg = (struct SRoU *)(rte_pktmbuf_mtod(pkt[i], char *) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr)); 126 | *msg = obj; 127 | int pkt_size = sizeof(struct SRoU) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr); 128 | 129 | //最后是采用HW Offload的方式去计算Checksum 130 | pkt[i]->l2_len = sizeof(struct rte_ether_hdr); 131 | pkt[i]->l3_len = sizeof(struct rte_ipv4_hdr); 132 | pkt[i]->l4_len = sizeof(struct rte_udp_hdr); 133 | pkt[i]->ol_flags |= PKT_TX_IPV4 | PKT_TX_IP_CKSUM | PKT_TX_UDP_CKSUM; 134 | 135 | ipv4_hdr->total_length = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr) + sizeof(struct rte_ipv4_hdr)); 136 | ipv4_hdr->hdr_checksum = 0; 137 | udp_hdr->dgram_cksum = rte_ipv4_phdr_cksum(ipv4_hdr, pkt[i]->ol_flags); 138 | 139 | //定义报文长度 140 | pkt[i]->data_len = pkt_size; 141 | pkt[i]->pkt_len = pkt_size; 142 | } 143 | ``` 144 | 然后我们采用每两秒发送一次的方式 145 | ```c 146 | for(;;) { 147 | uint16_t nb_tx = rte_eth_tx_burst(0, 0, pkt, BURST_SIZE); 148 | printf("successful send %d pkts\n", nb_tx); 149 | sleep(2); 150 | 151 | } 152 | for (int i = 0; i < BURST_SIZE; i++) 153 | { 154 | rte_pktmbuf_free(pkt[i]); 155 | } 156 | ``` 157 | 执行时，我们希望收发并行执行，因此我们可以将发包函数封装好, 并在main函数中调用 158 | ```c 159 | static int 160 | lcore_send(struct rte_mempool *mbuf_pool) { 161 | ... 162 | } 163 | 164 | int main(){ 165 | 166 | rte_eal_remote_launch((lcore_function_t *)lcore_send,mbuf_pool,1); 167 | 168 | } 169 | ``` -------------------------------------------------------------------------------- /01_port_init/devinfo/main.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #define MAX_PORTS 16 15 | 16 | typedef struct 17 | { 18 | uint64_t code; 19 | const char *name; 20 | } offload_capa_t; 21 | 22 | static offload_capa_t rx_capa_name[] = { 23 | {DEV_RX_OFFLOAD_VLAN_STRIP, "DEV_RX_OFFLOAD_VLAN_STRIP"}, 24 | {DEV_RX_OFFLOAD_IPV4_CKSUM, "DEV_RX_OFFLOAD_IPV4_CKSUM"}, 25 | {DEV_RX_OFFLOAD_UDP_CKSUM, "DEV_RX_OFFLOAD_UDP_CKSUM"}, 26 | {DEV_RX_OFFLOAD_TCP_CKSUM, "DEV_RX_OFFLOAD_TCP_CKSUM"}, 27 | {DEV_RX_OFFLOAD_TCP_LRO, "DEV_RX_OFFLOAD_TCP_LRO"}, 28 | {DEV_RX_OFFLOAD_QINQ_STRIP, "DEV_RX_OFFLOAD_QINQ_STRIP"}, 29 | {DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM, "DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM"}, 30 | {DEV_RX_OFFLOAD_MACSEC_STRIP, "DEV_RX_OFFLOAD_MACSEC_STRIP"}, 31 | {DEV_RX_OFFLOAD_HEADER_SPLIT, "DEV_RX_OFFLOAD_HEADER_SPLIT"}, 32 | {DEV_RX_OFFLOAD_VLAN_FILTER, "DEV_RX_OFFLOAD_VLAN_FILTER"}, 33 | {DEV_RX_OFFLOAD_VLAN_EXTEND, "DEV_RX_OFFLOAD_VLAN_EXTEND"}, 34 | {DEV_RX_OFFLOAD_JUMBO_FRAME, "DEV_RX_OFFLOAD_JUMBO_FRAME"}, 35 | {DEV_RX_OFFLOAD_SCATTER, "DEV_RX_OFFLOAD_SCATTER"}, 36 | {DEV_RX_OFFLOAD_TIMESTAMP, "DEV_RX_OFFLOAD_TIMESTAMP"}, 37 | {DEV_RX_OFFLOAD_SECURITY, "DEV_RX_OFFLOAD_SECURITY"}, 38 | {DEV_RX_OFFLOAD_KEEP_CRC, "DEV_RX_OFFLOAD_KEEP_CRC"}, 39 | {DEV_RX_OFFLOAD_SCTP_CKSUM, "DEV_RX_OFFLOAD_SCTP_CKSUM"}, 40 | {DEV_RX_OFFLOAD_OUTER_UDP_CKSUM, "DEV_RX_OFFLOAD_OUTER_UDP_CKSUM"}, 41 | {DEV_RX_OFFLOAD_RSS_HASH, "DEV_RX_OFFLOAD_RSS_HASH"}}; 42 | 43 | static offload_capa_t tx_capa_name[] = { 44 | {DEV_TX_OFFLOAD_VLAN_INSERT, "DEV_TX_OFFLOAD_VLAN_INSERT"}, 45 | {DEV_TX_OFFLOAD_IPV4_CKSUM, "DEV_TX_OFFLOAD_IPV4_CKSUM"}, 46 | {DEV_TX_OFFLOAD_UDP_CKSUM, "DEV_TX_OFFLOAD_UDP_CKSUM"}, 47 | {DEV_TX_OFFLOAD_TCP_CKSUM, "DEV_TX_OFFLOAD_TCP_CKSUM"}, 48 | {DEV_TX_OFFLOAD_SCTP_CKSUM, "DEV_TX_OFFLOAD_SCTP_CKSUM"}, 49 | {DEV_TX_OFFLOAD_TCP_TSO, "DEV_TX_OFFLOAD_TCP_TSO"}, 50 | {DEV_TX_OFFLOAD_UDP_TSO, "DEV_TX_OFFLOAD_UDP_TSO"}, 51 | {DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM, "DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM"}, 52 | {DEV_TX_OFFLOAD_QINQ_INSERT, "DEV_TX_OFFLOAD_QINQ_INSERT"}, 53 | {DEV_TX_OFFLOAD_VXLAN_TNL_TSO, "DEV_TX_OFFLOAD_VXLAN_TNL_TSO"}, 54 | {DEV_TX_OFFLOAD_GRE_TNL_TSO, "DEV_TX_OFFLOAD_GRE_TNL_TSO"}, 55 | {DEV_TX_OFFLOAD_IPIP_TNL_TSO, "DEV_TX_OFFLOAD_IPIP_TNL_TSO"}, 56 | {DEV_TX_OFFLOAD_GENEVE_TNL_TSO, "DEV_TX_OFFLOAD_GENEVE_TNL_TSO"}, 57 | {DEV_TX_OFFLOAD_MACSEC_INSERT, "DEV_TX_OFFLOAD_MACSEC_INSERT"}, 58 | {DEV_TX_OFFLOAD_MT_LOCKFREE, "DEV_TX_OFFLOAD_MT_LOCKFREE"}, 59 | {DEV_TX_OFFLOAD_MULTI_SEGS, "DEV_TX_OFFLOAD_MULTI_SEGS"}, 60 | {DEV_TX_OFFLOAD_MBUF_FAST_FREE, "DEV_TX_OFFLOAD_MBUF_FAST_FREE"}, 61 | {DEV_TX_OFFLOAD_SECURITY, "DEV_TX_OFFLOAD_SECURITY"}, 62 | {DEV_TX_OFFLOAD_UDP_TNL_TSO, "DEV_TX_OFFLOAD_UDP_TNL_TSO"}, 63 | {DEV_TX_OFFLOAD_IP_TNL_TSO, "DEV_TX_OFFLOAD_IP_TNL_TSO"}, 64 | {DEV_TX_OFFLOAD_OUTER_UDP_CKSUM, "DEV_TX_OFFLOAD_OUTER_UDP_CKSUM"}, 65 | {DEV_TX_OFFLOAD_SEND_ON_TIMESTAMP, "DEV_TX_OFFLOAD_SEND_ON_TIMESTAMP"}, 66 | }; 67 | 68 | int main(int argc, char *argv[]) 69 | { 70 | 71 | int ret = rte_eal_init(argc, argv); 72 | if (ret < 0) 73 | rte_exit(EXIT_FAILURE, "initlize fail!"); 74 | 75 | printf("\n\n\n*****************************************\n"); 76 | 77 | int nb_ports, port_id; 78 | 79 | nb_ports = rte_eth_dev_count_avail(); 80 | printf("number of available port: %d\n", nb_ports); 81 | 82 | struct rte_eth_dev_info dev_info; 83 | 84 | RTE_ETH_FOREACH_DEV(port_id) 85 | { 86 | printf("=========================================\n"); 87 | 88 | /* get device info */ 89 | ret = rte_eth_dev_info_get(port_id, &dev_info); 90 | if (ret != 0) 91 | { 92 | printf("Error during getting device (port %u) info: %s\n", 93 | port_id, strerror(-ret)); 94 | return ret; 95 | } 96 | 97 | printf("port: %d \tDriver:%s \n", port_id, dev_info.driver_name); 98 | struct rte_eth_link link; 99 | char link_status[RTE_ETH_LINK_MAX_STR_LEN]; 100 | ret = rte_eth_link_get_nowait(port_id, &link); 101 | rte_eth_link_to_str(link_status, sizeof(link_status), &link); 102 | printf("%s\n", link_status); 103 | 104 | char port_name[RTE_ETH_NAME_MAX_LEN]; 105 | ret = rte_eth_dev_get_name_by_port(port_id, port_name); 106 | if (ret < 0) 107 | rte_exit(EXIT_FAILURE, "Cannot get port name: err=%d, port=%d\n", ret, port_id); 108 | 109 | static struct rte_ether_addr ports_eth_addr; 110 | ret = rte_eth_macaddr_get(port_id, &ports_eth_addr); 111 | if (ret < 0) 112 | rte_exit(EXIT_FAILURE, "Cannot get MAC address: err=%d, port=%d\n", ret, port_id); 113 | 114 | char mac[18]; 115 | rte_ether_format_addr(&mac[0], 18, &ports_eth_addr); 116 | printf("MAC address: %s\nPCIe:%s\n", mac, port_name); 117 | 118 | printf("Max RX Queue: \t%d\tDesc:\t%d\n", dev_info.max_rx_queues, dev_info.rx_desc_lim.nb_max); 119 | printf("Max TX Queue: \t%d\tDesc:\t%d\n", dev_info.max_tx_queues, dev_info.tx_desc_lim.nb_max); 120 | 121 | printf("Offload Capability:\n"); 122 | /* check offload capability */ 123 | uint64_t rx_capa = dev_info.rx_offload_capa; 124 | uint32_t num = sizeof(rx_capa_name) / sizeof(offload_capa_t); 125 | for (int i = 0; i < num; ++i) 126 | { 127 | if (rx_capa & rx_capa_name[i].code) 128 | { 129 | printf(" %s\n", rx_capa_name[i].name); 130 | } 131 | } 132 | printf("-----------------------------------------\n"); 133 | 134 | uint64_t tx_capa = dev_info.tx_offload_capa; 135 | num = sizeof(tx_capa_name) / sizeof(offload_capa_t); 136 | for (int i = 0; i < num; ++i) 137 | { 138 | if (tx_capa & tx_capa_name[i].code) 139 | { 140 | printf(" %s\n", tx_capa_name[i].name); 141 | } 142 | } 143 | 144 | printf("=========================================\n\n"); 145 | } 146 | return 0; 147 | } 148 | -------------------------------------------------------------------------------- /a2_kvm/README.md: -------------------------------------------------------------------------------- 1 | ## 虚拟机基本管理 2 | 3 | 如下命令可以修改默认网段 4 | 5 | ```bash 6 | sudo virsh net-edit --network default 7 | ``` 8 | ```xml 9 | 10 | default 11 | 45ed012c-3933-4f3e-9575-b37bffa21b83 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | ``` 22 | Enable forwarding 23 | ```bash 24 | echo "net.ipv4.ip_forward=1" | sudo tee /etc/sysctl.conf 25 | ``` 26 | 27 | 28 | ```bash 29 | [root@netdev vm]# virsh list --all 30 | Id Name State 31 | ------------------------ 32 | 1 ubuntu running 33 | ``` 34 | 35 | 启动、关闭、删除 36 | virsh [start/shutdown/destroy] vm 37 | 38 | 挂起和恢复 39 | virsh [suspend|resume] vm 40 | 41 | 开机自启动 42 | virsh autostart vm 43 | 44 | 查看信息 45 | ```bash 46 | 47 | [root@netdev vm]# virsh dominfo ubuntu 48 | Id: 1 49 | Name: ubuntu 50 | UUID: dd1e0fee-0f38-4a9a-a515-256bb6a10d16 51 | OS Type: hvm 52 | State: running 53 | CPU(s): 4 54 | CPU time: 539.3s 55 | Max memory: 8388608 KiB 56 | Used memory: 8388608 KiB 57 | Persistent: yes 58 | Autostart: disable 59 | Managed save: no 60 | Security model: selinux 61 | Security DOI: 0 62 | Security label: system_u:system_r:svirt_t:s0:c887,c939 (enforcing) 63 | 64 | ``` 65 | 克隆虚机 66 | ```bash 67 | [root@netdev vm]# virt-clone -o ubuntu -n vm001 -f /home/vm/vm001.qcow2 68 | Allocating 'vm001.qcow2' | 100 GB 00:00:08 69 | 70 | Clone 'vm001' created successfully. 71 | ``` 72 | 克隆完成后， 需要uuidgen 73 | ```bash 74 | [root@netdev vm]# uuidgen 75 | 10c35319-bd71-4447-aa1a-88207ec42fbf 76 | [root@netdev vm]# vim /etc/libvirt/qemu/vm001.xml 77 | ``` 78 | 删除虚机 79 | ```bash 80 | virsh undefine vm 81 | ``` 82 | 83 | 删除虚机并删除存储 84 | ```bash 85 | virsh undefine vm --storage /home/vm/vm.qcow2 86 | virsh undefine vm --remove-all-storage 87 | ``` 88 | 89 | dump xml 90 | ```bash 91 | [root@netdev vm]# virsh dumpxml ubuntu 92 | ``` 93 | 94 | 95 | 使用root登录vnc，然后启用virt-manager 96 | 97 | 98 | ```bash 99 | virt-manager 100 | ``` 101 | ![image](https://github.com/zartbot/learn_dpdk/blob/main/a2_kvm/images/1.png?raw=true) 102 | 103 | 104 | 然后创建虚机过程就不多讲了，添加PCIe Device将Mellanox的VF网卡加入即可. 105 | 106 | ![image](https://github.com/zartbot/learn_dpdk/blob/main/a2_kvm/images/2.png?raw=true) 107 | 108 | 在VM中测试性能不好， 109 | 注意到这个错误提示 110 | ```bash 111 | EAL: Invalid NUMA socket, default to 0 112 | ``` 113 | 检查Numa 114 | ```bash 115 | [zartbot@netdev ~]$ sudo numactl --hardware 116 | available: 2 nodes (0-1) 117 | node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 118 | node 0 size: 176933 MB 119 | node 0 free: 174714 MB 120 | node 1 cpus: 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 121 | node 1 size: 182932 MB 122 | node 1 free: 152286 MB 123 | node distances: 124 | node 0 1 125 | 0: 10 21 126 | 1: 21 10 127 | ``` 128 | 129 | 检查VCPU分配 130 | 131 | ```bash 132 | [zartbot@netdev ~]$ sudo virsh vcpuinfo vm001 133 | VCPU: 0 134 | CPU: 75 135 | State: running 136 | CPU time: 806.6s 137 | CPU Affinity: yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy 138 | 139 | VCPU: 1 140 | CPU: 41 141 | State: running 142 | CPU time: 792.8s 143 | CPU Affinity: yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy 144 | 145 | VCPU: 2 146 | CPU: 39 147 | State: running 148 | CPU time: 689.6s 149 | CPU Affinity: yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy 150 | 151 | VCPU: 3 152 | CPU: 77 153 | State: running 154 | CPU time: 685.5s 155 | CPU Affinity: yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy 156 | ``` 157 | 以下关于CPU和内存的参数设定建议来自于 158 | ```mk 159 | https://libvirt.org/formatdomain.html 160 | https://libvirt.org/kbase/kvm-realtime.html 161 | ``` 162 | 163 | 注意修改 164 | ```bash 165 | virsh edit vm001 166 | ``` 167 | ```xml 168 | 169 | vm001 170 | a6125142-bc3b-4453-a010-2c03077e7e09 171 | 172 | 173 | 174 | 175 | 176 | 8388608 177 | 8388608 178 | 179 | 8 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | hvm 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | ``` 206 | 207 | 然后性能就正常了 208 | ```bash 209 | zartbot@zartbot-KVM:~/test$ sudo ./build/helloworld 210 | EAL: Detected 8 lcore(s) 211 | EAL: Detected 1 NUMA nodes 212 | EAL: Detected shared linkage of DPDK 213 | EAL: Multi-process socket /var/run/dpdk/rte/mp_socket 214 | EAL: Selected IOVA mode 'PA' 215 | EAL: No available 1048576 kB hugepages reported 216 | EAL: VFIO support initialized 217 | EAL: Invalid NUMA socket, default to 0 218 | EAL: Probe PCI driver: net_virtio (1af4:1041) device: 0000:01:00.0 (socket 0) 219 | eth_virtio_pci_init(): Failed to init PCI device 220 | 221 | EAL: Requested device 0000:01:00.0 cannot be used 222 | EAL: Invalid NUMA socket, default to 0 223 | EAL: Probe PCI driver: mlx5_pci (15b3:101a) device: 0000:06:00.0 (socket 0) 224 | EAL: Invalid NUMA socket, default to 0 225 | EAL: Probe PCI driver: mlx5_pci (15b3:101a) device: 0000:07:00.0 (socket 0) 226 | TELEMETRY: No legacy callbacks, legacy socket not created 227 | 228 | 229 | 230 | ***************************************** 231 | number of available port: 2 232 | 233 | 234 | initializing port 0... 235 | port[0] support RX cheksum offload. 236 | port[0] support TX mbuf fast free offload. 237 | port[0] support TX IPv4 checksum offload. 238 | port[0] support TX UDP checksum offload. 239 | port[0] support TX TCP checksum offload. 240 | Port[0] MAC: 7a:99:ed:5f:e3:a6 241 | 242 | 243 | initializing port 1... 244 | port[1] support RX cheksum offload. 245 | port[1] support TX mbuf fast free offload. 246 | port[1] support TX IPv4 checksum offload. 247 | port[1] support TX UDP checksum offload. 248 | port[1] support TX TCP checksum offload. 249 | Port[1] MAC: 5a:d8:51:db:17:2d 250 | PPS: 30 251 | PPS: 29593410 252 | PPS: 29587011 253 | PPS: 29777849 254 | PPS: 29716992 255 | PPS: 29693889 256 | PPS: 29798295 257 | ``` 258 | -------------------------------------------------------------------------------- /04_rss/README.md: -------------------------------------------------------------------------------- 1 | 2 | > 前面一个示例只有单线程的收发，如果利用一个网卡进行多线程的收发呢？ 主要是网卡的TX多队列，和RX多队列，具体内容参见github上`04_rss`内的代码. 3 | 4 | ## 网卡支持情况 5 | 6 | 可以通过以下命令查看 7 | ```bash 8 | //队列数 9 | [zartbot@netdev multiThreadRing]$ ethtool -l ens17f0 10 | Channel parameters for ens17f0: 11 | Pre-set maximums: 12 | RX: 0 13 | TX: 0 14 | Other: 512 15 | Combined: 63 16 | Current hardware settings: 17 | RX: 0 18 | TX: 0 19 | Other: 0 20 | Combined: 63 21 | 22 | //队列深度 23 | [zartbot@netdev multiThreadRing]$ ethtool -g ens17f0 24 | Ring parameters for ens17f0: 25 | Pre-set maximums: 26 | RX: 8192 27 | RX Mini: 0 28 | RX Jumbo: 0 29 | TX: 8192 30 | Current hardware settings: 31 | RX: 1024 32 | RX Mini: 0 33 | RX Jumbo: 0 34 | TX: 1024 35 | ``` 36 | 37 | ## 多个TX 38 | 39 | 多个TX Queue的配置相对容易，在`port_init`函数中初始化即可 40 | ```c 41 | for (q = 0; q < tx_rings; q++) 42 | { 43 | retval = rte_eth_tx_queue_setup(port, q, nb_txd, 44 | rte_eth_dev_socket_id(port), &txconf); 45 | if (retval < 0) 46 | return retval; 47 | } 48 | ``` 49 | 然后我们会在main函数中为每个Queue创建一个Ring 50 | ```c 51 | for (int i = 0; i < NUM_TX_QUEUE; ++i) 52 | { 53 | char tx_ring_name[14]; 54 | sprintf(tx_ring_name, "output_ring%d", i); 55 | tx_ring[i] = rte_ring_create(tx_ring_name, SCHED_TX_RING_SZ, 56 | rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); 57 | if (tx_ring[i] == NULL) 58 | rte_exit(EXIT_FAILURE, "Cannot create output ring\n"); 59 | } 60 | ``` 61 | 然后调用不同的lcore发包，这只是一个1:1的lcore_tx和lcore_send_pkt示例，通常可以让一个lcore去同时处理多个ring，并执行发包程序. 62 | ```c 63 | 64 | struct lcore_params { 65 | struct rte_ring *rx_ring; 66 | struct rte_ring *tx_ring; 67 | uint16_t rx_queue_id; 68 | uint16_t tx_queue_id; 69 | struct rte_mempool *mem_pool; 70 | }; 71 | 72 | /* Start rx core */ 73 | for (int i = 0; i < NUM_RX_QUEUE; ++i) 74 | { 75 | lp_rx[i] = rte_malloc(NULL, sizeof(*lp_rx[i]), 0); 76 | if (!lp_rx[i]) 77 | rte_panic("malloc failure\n"); 78 | *lp_rx[i] = (struct lcore_params){rx_ring[i], NULL, i, i, mbuf_pool}; 79 | rte_eal_remote_launch((lcore_function_t *)lcore_rx, lp_rx[i], lcore_num++); 80 | 81 | //start pkt dequeue process 82 | rxp[i] = rte_malloc(NULL, sizeof(*rxp[i]), 0); 83 | if (!rxp[i]) 84 | rte_panic("malloc failure\n"); 85 | *rxp[i] = (struct rx_params){i,rx_ring[i]}; 86 | 87 | rte_eal_remote_launch((lcore_function_t *)lcore_recv_pkt, rxp[i], lcore_num++); 88 | } 89 | 90 | ``` 91 | 在发包的时候，由于queue_id通过lcore_params传入，因此tx函数如下 92 | ```c 93 | 94 | int lcore_tx(struct lcore_params *p) 95 | { 96 | uint16_t port = 0; 97 | printf("Core %u doing packet TX.\n", rte_lcore_id()); 98 | struct rte_mbuf *bufs[BURST_SIZE_TX]; 99 | while (1) 100 | { 101 | const uint16_t nb_rx = rte_ring_dequeue_burst(p->tx_ring, 102 | (void *)bufs, BURST_SIZE_TX, NULL); 103 | if (unlikely(nb_rx == 0)) 104 | { 105 | 106 | continue; 107 | } 108 | //发包函数带上了queue_id 109 | unsigned int nb_tx = rte_eth_tx_burst(port, p->tx_queue_id, bufs, nb_rx); 110 | if (unlikely(nb_tx < nb_rx)) { 111 | do 112 | { 113 | rte_pktmbuf_free(bufs[nb_tx]); 114 | } while(++nb_tx < nb_rx); 115 | } 116 | } 117 | return 0; 118 | } 119 | ``` 120 | 121 | ## RSS 122 | 123 | RSS是Recieve Side Scaling的意思，即通过网络数据包的一些字段构成hash值，然后分发到不同的接收队列上.网卡的支持情况如下可以查看 124 | ```bash 125 | [zartbot@netdev multiThreadRing]$ ethtool -x ens17f0 126 | RX flow hash indirection table for ens17f0 with 63 RX ring(s): 127 | 0: 0 1 2 3 4 5 6 7 128 | 8: 8 9 10 11 12 13 14 15 129 | 16: 16 17 18 19 20 21 22 23 130 | 24: 24 25 26 27 28 29 30 31 131 | 32: 32 33 34 35 36 37 38 39 132 | 40: 40 41 42 43 44 45 46 47 133 | 48: 48 49 50 51 52 53 54 55 134 | 56: 56 57 58 59 60 61 62 0 135 | 64: 1 2 3 4 5 6 7 8 136 | 72: 9 10 11 12 13 14 15 16 137 | 80: 17 18 19 20 21 22 23 24 138 | 88: 25 26 27 28 29 30 31 32 139 | 96: 33 34 35 36 37 38 39 40 140 | 104: 41 42 43 44 45 46 47 48 141 | 112: 49 50 51 52 53 54 55 56 142 | 120: 57 58 59 60 61 62 0 1 143 | RSS hash key: 144 | f4:8a:21:18:f8:03:b2:75:17:45:c5:3b:c7:db:66:b4:eb:73:ea:53:7d:13:93:3b:f1:50:65:55:e1:20:d6:3d:87:52:59:f3:f8:25:f2:2c 145 | RSS hash function: 146 | toeplitz: on 147 | xor: off 148 | crc32: off 149 | ``` 150 | 我们需要在port_init的阶段配置RSS 151 | ```c 152 | static const struct rte_eth_conf port_conf_default = { 153 | .rxmode = { 154 | .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 155 | .mq_mode = ETH_MQ_RX_RSS, 156 | }, 157 | .rx_adv_conf = { 158 | .rss_conf = { 159 | .rss_key = NULL, 160 | .rss_hf = ETH_RSS_IP | ETH_RSS_TCP | ETH_RSS_UDP, 161 | }, 162 | }, 163 | .txmode = { 164 | .mq_mode = ETH_MQ_TX_NONE, 165 | } 166 | }; 167 | ``` 168 | 然后同样也是创建多个`rx_queue` 169 | ```c 170 | for (q = 0; q < rx_rings; q++) 171 | { 172 | retval = rte_eth_rx_queue_setup(port, q, nb_rxd, 173 | rte_eth_dev_socket_id(port), NULL, mbuf_pool); 174 | if (retval < 0) 175 | return retval; 176 | } 177 | ``` 178 | 和上一讲区别就是在收包函数中指定了队列 179 | ```c 180 | const uint16_t nb_rx = rte_eth_rx_burst(port, p->rx_queue_id, bufs, 181 | BURST_SIZE); 182 | ``` 183 | 184 | 最终我们可以8个TX， 4个RX, 56Mpps... 185 | ```c 186 | EAL: Detected 96 lcore(s) 187 | EAL: Detected 2 NUMA nodes 188 | EAL: Detected shared linkage of DPDK 189 | EAL: Multi-process socket /var/run/dpdk/rte/mp_socket 190 | EAL: Selected IOVA mode 'VA' 191 | EAL: No available 1048576 kB hugepages reported 192 | EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0000:86:00.0 (socket 1) 193 | EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0000:86:00.1 (socket 1) 194 | TELEMETRY: No legacy callbacks, legacy socket not created 195 | 196 | 197 | initializing port 0... 198 | port[0] support RX cheksum offload. 199 | port[0] support TX mbuf fast free offload. 200 | port[0] support TX IPv4 checksum offload. 201 | port[0] support TX UDP checksum offload. 202 | port[0] support TX TCP checksum offload. 203 | Port[0] MAC: ec:0d:9a:c5:df:fc 204 | 205 | 206 | initializing port 1... 207 | port[1] support RX cheksum offload. 208 | port[1] support TX mbuf fast free offload. 209 | port[1] support TX IPv4 checksum offload. 210 | port[1] support TX UDP checksum offload. 211 | port[1] support TX TCP checksum offload. 212 | Port[1] MAC: ec:0d:9a:c5:df:fd 213 | Core 1 doing packet RX. 214 | Core 2 doing RX dequeue. 215 | Core 3 doing packet RX. 216 | Core 4 doing RX dequeue. 217 | Core 5 doing packet RX. 218 | Core 6 doing RX dequeue. 219 | Core 7 doing packet RX. 220 | Core 8 doing RX dequeue. 221 | Core 9 doing packet TX. 222 | Core 10 doing packet enqueue. 223 | Core 11 doing packet TX. 224 | Core 12 doing packet enqueue. 225 | Core 13 doing packet TX. 226 | Core 14 doing packet enqueue. 227 | Core 15 doing packet TX. 228 | Core 16 doing packet enqueue. 229 | Core 17 doing packet TX. 230 | Core 18 doing packet enqueue. 231 | Core 19 doing packet TX. 232 | Core 20 doing packet enqueue. 233 | Core 21 doing packet TX. 234 | Core 22 doing packet enqueue. 235 | Core 23 doing packet TX. 236 | Core 24 doing packet enqueue. 237 | RX-Queue[0] PPS: 14097659 238 | RX-Queue[1] PPS: 14100129 239 | RX-Queue[2] PPS: 14087252 240 | RX-Queue[3] PPS: 14089980 241 | 242 | RX-Queue[0] PPS: 14124369 243 | RX-Queue[1] PPS: 14124095 244 | RX-Queue[2] PPS: 14109246 245 | RX-Queue[3] PPS: 14109280 246 | 247 | RX-Queue[0] PPS: 14122728 248 | RX-Queue[1] PPS: 14122696 249 | RX-Queue[2] PPS: 14107982 250 | RX-Queue[3] PPS: 14107952 251 | 252 | RX-Queue[0] PPS: 14123040 253 | RX-Queue[1] PPS: 14122902 254 | RX-Queue[2] PPS: 14108187 255 | RX-Queue[3] PPS: 14108188 256 | ``` 257 | ## 展望 258 | 259 | 如何有效的调度报文和如何安排多个core中的功能便是写dataplane的艺术了.当然很多时候在设计协议时，也需要考虑到不同的硬件实现，使得协议处理更加高效. 260 | -------------------------------------------------------------------------------- /01_port_init/README.md: -------------------------------------------------------------------------------- 1 | ## 配置端口 2 | 3 | 首先需要初始化EAL 4 | ```c 5 | int ret = rte_eal_init(argc, argv); 6 | if (ret < 0) 7 | rte_exit(EXIT_FAILURE, "initlize fail!"); 8 | ``` 9 | 10 | 然后通过`rte_eth_dev_count_avail()`函数获取系统的接口数目: 11 | ```c 12 | int nb_ports; 13 | nb_ports = rte_eth_dev_count_avail(); 14 | printf("number of available port: %d\n", nb_ports); 15 | ``` 16 | 然后我们可以通过rte_eth_dev_info_get如下方式获取device info： 17 | ```c 18 | struct rte_eth_dev_info dev_info; 19 | for (int portid = 0; portid < nb_ports; ++portid) 20 | { 21 | ret = rte_eth_dev_info_get(portid, &dev_info); 22 | if (ret < 0) 23 | rte_exit(EXIT_FAILURE, "Cannot get device info: err=%d, port=%d\n", ret, portid); 24 | printf("port: %d Driver:%s\n", portid, dev_info.driver_name); 25 | } 26 | ``` 27 | 最后可以根据rte_eth_macaddr_get函数获取接口MAC地址,并放在ports_eth_addr数组中. 28 | ```c 29 | static struct rte_ether_addr ports_eth_addr[MAX_PORTS]; 30 | for (int portid = 0; portid < nb_ports; ++portid) 31 | { 32 | ret = rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); 33 | if (ret < 0) 34 | rte_exit(EXIT_FAILURE, "Cannot get MAC address: err=%d, port=%d\n", ret, portid); 35 | 36 | char mac[18]; 37 | rte_ether_format_addr(&mac[0], 18, &ports_eth_addr[portid]); 38 | printf("port: %d->MAC-> %s\n", portid, mac); 39 | } 40 | ``` 41 | 42 | ## 接口支持能力 43 | 44 | 更详细的devinfo处理可以参考01_port_init/devinfo. 45 | 46 | ```bash 47 | [root@c220m4-Gen2 devinfo]# sudo ./build/devinfo 48 | EAL: Detected 72 lcore(s) 49 | EAL: Detected 2 NUMA nodes 50 | EAL: Detected shared linkage of DPDK 51 | EAL: Multi-process socket /var/run/dpdk/rte/mp_socket 52 | EAL: Selected IOVA mode 'VA' 53 | EAL: No available 1048576 kB hugepages reported 54 | EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0000:0e:00.0 (socket 0) 55 | mlx5_net: No available register for sampler. 56 | EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0000:0e:00.1 (socket 0) 57 | mlx5_net: No available register for sampler. 58 | EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0000:81:00.0 (socket 1) 59 | mlx5_net: No available register for sampler. 60 | EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0000:81:00.1 (socket 1) 61 | mlx5_net: No available register for sampler. 62 | TELEMETRY: No legacy callbacks, legacy socket not created 63 | 64 | 65 | 66 | ***************************************** 67 | number of available port: 4 68 | ========================================= 69 | port: 0 Driver:mlx5_pci 70 | Link up at 100 Gbps FDX Autoneg 71 | MAC address: 04:3F:72:EA:E2:00 72 | PCIe:0000:0e:00.0 73 | Max RX Queue: 1024 Desc: 65535 74 | Max TX Queue: 1024 Desc: 65535 75 | Offload Capability: 76 | DEV_RX_OFFLOAD_VLAN_STRIP 77 | DEV_RX_OFFLOAD_IPV4_CKSUM 78 | DEV_RX_OFFLOAD_UDP_CKSUM 79 | DEV_RX_OFFLOAD_TCP_CKSUM 80 | DEV_RX_OFFLOAD_TCP_LRO 81 | DEV_RX_OFFLOAD_VLAN_FILTER 82 | DEV_RX_OFFLOAD_JUMBO_FRAME 83 | DEV_RX_OFFLOAD_SCATTER 84 | DEV_RX_OFFLOAD_TIMESTAMP 85 | DEV_RX_OFFLOAD_KEEP_CRC 86 | DEV_RX_OFFLOAD_RSS_HASH 87 | ----------------------------------------- 88 | DEV_TX_OFFLOAD_VLAN_INSERT 89 | DEV_TX_OFFLOAD_IPV4_CKSUM 90 | DEV_TX_OFFLOAD_UDP_CKSUM 91 | DEV_TX_OFFLOAD_TCP_CKSUM 92 | DEV_TX_OFFLOAD_TCP_TSO 93 | DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM 94 | DEV_TX_OFFLOAD_VXLAN_TNL_TSO 95 | DEV_TX_OFFLOAD_GRE_TNL_TSO 96 | DEV_TX_OFFLOAD_GENEVE_TNL_TSO 97 | DEV_TX_OFFLOAD_MULTI_SEGS 98 | DEV_TX_OFFLOAD_MBUF_FAST_FREE 99 | DEV_TX_OFFLOAD_UDP_TNL_TSO 100 | DEV_TX_OFFLOAD_IP_TNL_TSO 101 | ========================================= 102 | 103 | ``` 104 | 105 | ## 复杂一点的接口初始化 106 | 107 | 在需要用DPDK收发包时，通常我们需要做更复杂的接口初始化操作，因此我们通常会专门写一个port_init函数,这个函数的参数为portid和相关的mbuf_pool： 108 | 109 | ```c 110 | static inline int 111 | port_init(uint16_t port, struct rte_mempool *mbuf_pool) 112 | { 113 | ... 114 | } 115 | 116 | ``` 117 | 118 | 首先我们需要定义一些常量， 主要是RX_RING / TX_RING的大小， MBUF的大小和Cache_size等 119 | 120 | ```c 121 | #define RX_RING_SIZE 1024 122 | #define TX_RING_SIZE 1024 123 | #define NUM_MBUFS 8191 124 | #define MBUF_CACHE_SIZE 250 125 | #define BURST_SIZE 32 126 | ``` 127 | 128 | 接着定义一个default config的结构体 129 | ```c 130 | static const struct rte_eth_conf port_conf_default = { 131 | .rxmode = { 132 | .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 133 | }, 134 | }; 135 | ``` 136 | 接下来就是整个port_init函数了： 137 | 138 | ```c 139 | static inline int 140 | port_init(uint16_t port, struct rte_mempool *mbuf_pool) 141 | { 142 | struct rte_eth_conf port_conf = port_conf_default; 143 | const uint16_t rx_rings = 1, tx_rings = 1; 144 | uint16_t nb_rxd = RX_RING_SIZE; 145 | uint16_t nb_txd = TX_RING_SIZE; 146 | int retval; 147 | uint16_t q; 148 | struct rte_eth_dev_info dev_info; 149 | struct rte_eth_txconf txconf; 150 | 151 | //查看这个接口是否为valid，非法则返回-1 152 | if (!rte_eth_dev_is_valid_port(port)) 153 | return -1; 154 | 155 | //获取接口信息 156 | retval = rte_eth_dev_info_get(port, &dev_info); 157 | if (retval != 0) 158 | { 159 | printf("Error during getting device (port %u) info: %s\n", 160 | port, strerror(-retval)); 161 | return retval; 162 | } 163 | printf("\n\ninitializing port %d...\n",port); 164 | 165 | //查看接口硬件Offload的能力是否支持，如果支持打开该功能 166 | if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_CHECKSUM) 167 | { 168 | printf("port[%u] support RX cheksum offload.\n", port); 169 | port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CHECKSUM; 170 | } 171 | 172 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 173 | { 174 | printf("port[%u] support TX mbuf fast free offload.\n", port); 175 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE; 176 | } 177 | 178 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 179 | { 180 | printf("port[%u] support TX IPv4 checksum offload.\n", port); 181 | 182 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 183 | } 184 | 185 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) 186 | { 187 | printf("port[%u] support TX UDP checksum offload.\n", port); 188 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_UDP_CKSUM; 189 | } 190 | 191 | //配置接口 192 | retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); 193 | if (retval != 0) 194 | return retval; 195 | 196 | retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); 197 | if (retval != 0) 198 | return retval; 199 | 200 | //分配RX队列 201 | for (q = 0; q < rx_rings; q++) 202 | { 203 | retval = rte_eth_rx_queue_setup(port, q, nb_rxd, 204 | rte_eth_dev_socket_id(port), NULL, mbuf_pool); 205 | if (retval < 0) 206 | return retval; 207 | } 208 | 209 | txconf = dev_info.default_txconf; 210 | txconf.offloads = port_conf.txmode.offloads; 211 | 212 | //分配TX队列 213 | for (q = 0; q < tx_rings; q++) 214 | { 215 | retval = rte_eth_tx_queue_setup(port, q, nb_txd, 216 | rte_eth_dev_socket_id(port), &txconf); 217 | if (retval < 0) 218 | return retval; 219 | } 220 | 221 | //使能接口 222 | retval = rte_eth_dev_start(port); 223 | if (retval < 0) 224 | return retval; 225 | 226 | //获取接口MAC地址 227 | struct rte_ether_addr addr; 228 | retval = rte_eth_macaddr_get(port, &addr); 229 | if (retval != 0) 230 | return retval; 231 | 232 | printf("Port[%u] MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 233 | " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", 234 | port, 235 | addr.addr_bytes[0], addr.addr_bytes[1], 236 | addr.addr_bytes[2], addr.addr_bytes[3], 237 | addr.addr_bytes[4], addr.addr_bytes[5]); 238 | 239 | //打开混杂模式 240 | retval = rte_eth_promiscuous_enable(port); 241 | if (retval != 0) 242 | return retval; 243 | 244 | return 0; 245 | } 246 | ``` 247 | 然后main函数就很简单的了, 初始化EAL，然后创建Mbuf pool 248 | ```c 249 | mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports, 250 | MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 251 | if (mbuf_pool == NULL) 252 | rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 253 | ``` 254 | 然后初始化接口时，可以使用一个宏RTE_ETH_FOREACH_DEV(portid)将每个接口使能. 255 | 256 | ``` 257 | int main(int argc, char *argv[]) 258 | { 259 | 260 | struct rte_mempool *mbuf_pool; 261 | unsigned nb_ports; 262 | uint16_t portid; 263 | 264 | int ret = rte_eal_init(argc, argv); 265 | if (ret < 0) 266 | rte_exit(EXIT_FAILURE, "initlize fail!"); 267 | 268 | printf("\n\n\n*****************************************\n"); 269 | 270 | nb_ports = rte_eth_dev_count_avail(); 271 | printf("number of available port: %d\n", nb_ports); 272 | 273 | /* Creates a new mempool in memory to hold the mbufs. */ 274 | mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports, 275 | MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 276 | 277 | if (mbuf_pool == NULL) 278 | rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 279 | 280 | /* Initialize all ports. */ 281 | RTE_ETH_FOREACH_DEV(portid) 282 | if (port_init(portid, mbuf_pool) != 0) 283 | rte_exit(EXIT_FAILURE, "Cannot init port %" PRIu16 "\n", 284 | portid); 285 | 286 | return 0; 287 | } 288 | ``` 289 | 290 | 291 | 292 | -------------------------------------------------------------------------------- /03_ring/README.md: -------------------------------------------------------------------------------- 1 | ## 无锁队列概述 2 | 3 | DPDK中使用了一套无锁的环形缓冲队列机制`rte_ring`,它是一个固定长度的队列，主要包含如下功能. 4 | 5 | - FIFO (First In First Out) 6 | - Maximum size is fixed; the pointers are stored in a table. 7 | - Lockless implementation. 8 | - Multi- or single-consumer dequeue. 9 | - Multi- or single-producer enqueue. 10 | - Bulk dequeue. 11 | - Bulk enqueue. 12 | - Ability to select different sync modes for producer/consumer. 13 | - Dequeue start/finish (depending on consumer sync modes). 14 | - Enqueue start/finish (depending on producer sync mode). 15 | 16 | 详细文档可以参考： 17 | ```mk 18 | http://doc.dpdk.org/guides/prog_guide/ring_lib.html 19 | ``` 20 | 21 | ring的结构如下 22 | 23 | ![Ring Struct](https://github.com/zartbot/learn_dpdk/blob/main/03_ring/images/1.png?raw=true) 24 | 25 | ```c 26 | struct rte_ring { 27 | /* 28 | * Note: this field kept the RTE_MEMZONE_NAMESIZE size due to ABI 29 | * compatibility requirements, it could be changed to RTE_RING_NAMESIZE 30 | * next time the ABI changes 31 | */ 32 | char name[RTE_MEMZONE_NAMESIZE] __rte_cache_aligned; 33 | /**< Name of the ring. */ 34 | int flags; /**< Flags supplied at creation. */ 35 | const struct rte_memzone *memzone; 36 | /**< Memzone, if any, containing the rte_ring */ 37 | uint32_t size; /**< Size of ring. */ 38 | uint32_t mask; /**< Mask (size-1) of ring. */ 39 | uint32_t capacity; /**< Usable size of ring */ 40 | 41 | char pad0 __rte_cache_aligned; /**< empty cache line */ 42 | 43 | /** Ring producer status. */ 44 | RTE_STD_C11 45 | union { 46 | struct rte_ring_headtail prod; 47 | struct rte_ring_hts_headtail hts_prod; 48 | struct rte_ring_rts_headtail rts_prod; 49 | } __rte_cache_aligned; 50 | 51 | char pad1 __rte_cache_aligned; /**< empty cache line */ 52 | 53 | /** Ring consumer status. */ 54 | RTE_STD_C11 55 | union { 56 | struct rte_ring_headtail cons; 57 | struct rte_ring_hts_headtail hts_cons; 58 | struct rte_ring_rts_headtail rts_cons; 59 | } __rte_cache_aligned; 60 | 61 | char pad2 __rte_cache_aligned; /**< empty cache line */ 62 | }; 63 | ``` 64 | 里面有一个union，包含三种结构： 65 | ```c 66 | union { 67 | struct rte_ring_headtail prod; 68 | struct rte_ring_hts_headtail hts_prod; 69 | struct rte_ring_rts_headtail rts_prod; 70 | } 71 | ``` 72 | 后两种是多线程使用的Relaxed tail sync(rts)和head/tail sync(hts)暂且不管，我们来看最基本的`rte_ring_headtail`结构 73 | ```c 74 | struct rte_ring_headtail { 75 | volatile uint32_t head; /**< prod/consumer head. */ 76 | volatile uint32_t tail; /**< prod/consumer tail. */ 77 | RTE_STD_C11 78 | union { 79 | /** sync type of prod/cons */ 80 | enum rte_ring_sync_type sync_type; 81 | /** deprecated - True if single prod/cons */ 82 | uint32_t single; 83 | }; 84 | }; 85 | ``` 86 | 87 | 简单而言，就是这个队列包含四个指针，生产者的头尾指针(prod head/tail)和消费者的头尾指针(consumer head/tail) 88 | 89 | 由于这个队列可以同时支持 单生产者->单消费者(spsc),单生产者->多消费者(spmc),多生产者->单消费者(mpsc),多生产者->多消费者(mpmc)等几种模式，接下来我们先从最简单的spsc讲起 90 | 91 | ## 队列行为 92 | 93 | ### 单个生产者入队列 94 | 95 | ```c 96 | static __rte_always_inline unsigned int 97 | __rte_ring_do_enqueue_elem(struct rte_ring *r, const void *obj_table, 98 | unsigned int esize, unsigned int n, 99 | enum rte_ring_queue_behavior behavior, unsigned int is_sp, 100 | unsigned int *free_space) 101 | { 102 | uint32_t prod_head, prod_next; 103 | uint32_t free_entries; 104 | 105 | n = __rte_ring_move_prod_head(r, is_sp, n, behavior, 106 | &prod_head, &prod_next, &free_entries); 107 | if (n == 0) 108 | goto end; 109 | 110 | __rte_ring_enqueue_elems(r, prod_head, obj_table, esize, n); 111 | 112 | __rte_ring_update_tail(&r->prod, prod_head, prod_next, is_sp, 1); 113 | end: 114 | if (free_space != NULL) 115 | *free_space = free_entries - n; 116 | return n; 117 | } 118 | ``` 119 | ![Enqueue-1](https://github.com/zartbot/learn_dpdk/blob/main/03_ring/images/2.png?raw=true) 120 | 121 | 首先是把`prod_head`和`cons_tail`交给临时变量，`prod_next`指针向队列中的下一个对象，并采用检查`cons_tail`来看是否有足够的空间， 如果没有猪狗的空间，则跳转到`end`。 122 | 123 | 124 | ![Enqueue-2](https://github.com/zartbot/learn_dpdk/blob/main/03_ring/images/3.png?raw=true) 125 | 126 | 然后是`__rte_ring_enqueue_elems`保存对象到当前的`prod_head`位置， 最后是采用`__rte_ring_update_tail`将`prod_head`移动到`prod_next`完成入队操作. 127 | 128 | ![Enqueue-3](https://github.com/zartbot/learn_dpdk/blob/main/03_ring/images/4.png?raw=true) 129 | 130 | 131 | ### 单消费者出队 132 | 133 | ```c 134 | static __rte_always_inline unsigned int 135 | __rte_ring_do_dequeue_elem(struct rte_ring *r, void *obj_table, 136 | unsigned int esize, unsigned int n, 137 | enum rte_ring_queue_behavior behavior, unsigned int is_sc, 138 | unsigned int *available) 139 | { 140 | uint32_t cons_head, cons_next; 141 | uint32_t entries; 142 | 143 | n = __rte_ring_move_cons_head(r, (int)is_sc, n, behavior, 144 | &cons_head, &cons_next, &entries); 145 | if (n == 0) 146 | goto end; 147 | 148 | __rte_ring_dequeue_elems(r, cons_head, obj_table, esize, n); 149 | 150 | __rte_ring_update_tail(&r->cons, cons_head, cons_next, is_sc, 0); 151 | 152 | end: 153 | if (available != NULL) 154 | *available = entries - n; 155 | return n; 156 | } 157 | ``` 158 | ![dequeue-1](https://github.com/zartbot/learn_dpdk/blob/main/03_ring/images/5.png?raw=true) 159 | 160 | 161 | 同样也是将`cons_head`和`prod_tail`交给临时变量，然后`__rte_ring_move_cons_head`前移`cons_head`并指向`cons_next`，通过检查`prod_tail`查看队列是否为空，如果为空就跳转到`end`退出.否则就进行`__rte_ring_dequeue_elems`取出对象. 162 | ![dequeue-2](https://github.com/zartbot/learn_dpdk/blob/main/03_ring/images/6.png?raw=true) 163 | 164 | 取完了，更新`cons_tail`即可. 165 | 166 | ![dequeue-3](https://github.com/zartbot/learn_dpdk/blob/main/03_ring/images/7.png?raw=true) 167 | 168 | 169 | ### 多生产者入队 170 | 171 | 当有两个core需要入队时，此时需要一个`CAS`操作 172 | 173 | ![mp](https://github.com/zartbot/learn_dpdk/blob/main/03_ring/images/8.png?raw=true) 174 | 175 | 176 | ```c 177 | static __rte_always_inline unsigned int 178 | __rte_ring_move_prod_head(struct rte_ring *r, unsigned int is_sp, 179 | unsigned int n, enum rte_ring_queue_behavior behavior, 180 | uint32_t *old_head, uint32_t *new_head, 181 | uint32_t *free_entries) 182 | { 183 | const uint32_t capacity = r->capacity; 184 | unsigned int max = n; 185 | int success; 186 | 187 | do { 188 | /* Reset n to the initial burst count */ 189 | n = max; 190 | 191 | *old_head = r->prod.head; 192 | 193 | /* add rmb barrier to avoid load/load reorder in weak 194 | * memory model. It is noop on x86 195 | */ 196 | rte_smp_rmb(); 197 | 198 | /* 199 | * The subtraction is done between two unsigned 32bits value 200 | * (the result is always modulo 32 bits even if we have 201 | * *old_head > cons_tail). So 'free_entries' is always between 0 202 | * and capacity (which is < size). 203 | */ 204 | *free_entries = (capacity + r->cons.tail - *old_head); 205 | 206 | /* check that we have enough room in ring */ 207 | if (unlikely(n > *free_entries)) 208 | n = (behavior == RTE_RING_QUEUE_FIXED) ? 209 | 0 : *free_entries; 210 | 211 | if (n == 0) 212 | return 0; 213 | 214 | *new_head = *old_head + n; 215 | if (is_sp) 216 | r->prod.head = *new_head, success = 1; 217 | else 218 | success = rte_atomic32_cmpset(&r->prod.head, 219 | *old_head, *new_head); 220 | } while (unlikely(success == 0)); 221 | return n; 222 | } 223 | ``` 224 | 可以看到在`__rte_ring_move_prod_head`函数中，如果是MP模式，则采用原子操作去确保入队更新 225 | ```c 226 | success = rte_atomic32_cmpset(&r->prod.head, 227 | *old_head, *new_head); 228 | ``` 229 | 同时在update `prod_tail`时，也采用类似的操作去等待 230 | ```c 231 | if (!single) 232 | while (unlikely(ht->tail != old_val)) 233 | rte_pause(); 234 | 235 | ht->tail = new_val; 236 | ``` 237 | 238 | ![mp](https://github.com/zartbot/learn_dpdk/blob/main/03_ring/images/9.png?raw=true) 239 | 240 | 241 | >所以您需要注意，在使用MP、MC时性能会有一些影响,尽量在软件设计上使用SP、SC，当然还有一些保序的问题，留到讲DLB的时候. 顺便说一下，我发现一个很好的`SoC`, intel Atom凌动 P5962B, 有QAT又有内置的以太网控制器，还带新出的DLB，看上去很不错. 242 | 243 | 244 | ## 队列操作实战 245 | 246 | 示例在github同目录下的`simpleRing`中，首先在main函数里面创建了mempool和ring 247 | ```c 248 | zartbot_ring = rte_ring_create(_Z_RING, ring_size, rte_socket_id(), flags); 249 | message_pool = rte_mempool_create(_MSG_POOL, pool_size, 250 | 128, pool_cache, priv_data_sz, 251 | NULL, NULL, NULL, NULL, 252 | rte_socket_id(), flags); 253 | 254 | if (zartbot_ring == NULL) 255 | rte_exit(EXIT_FAILURE, "Problem getting receiving ring\n"); 256 | if (message_pool == NULL) 257 | rte_exit(EXIT_FAILURE, "Problem getting message pool\n"); 258 | ``` 259 | 260 | mempool和ring都有一个string可以在其它进程中索引获得， 我们在第二个示例中会介绍到,您先留意到这两个常量: 261 | 262 | ```c 263 | static const char *_MSG_POOL = "MSG_POOL"; 264 | static const char *_Z_RING = "ZARTBOT_RING"; 265 | ``` 266 | 然后我们分别使用不同的`lcore`发送和接收消息， 发送端主要是采用`rte_ring_enqueue`函数 267 | ```c 268 | static int 269 | lcore_send(__rte_unused void *arg) 270 | { 271 | unsigned lcore_id = rte_lcore_id(); 272 | struct rte_ring *tx_ring; 273 | printf("Starting core %u\n", lcore_id); 274 | void *msg = NULL; 275 | if (rte_mempool_get(message_pool, &msg) < 0) 276 | rte_panic("Failed to get message buffer\n"); 277 | 278 | for (int i=0;i<100;i++) { 279 | strlcpy((char *)msg, "hello", 128); 280 | if (rte_ring_enqueue(zartbot_ring, msg) < 0) { 281 | printf("Failed to send message - message discarded\n"); 282 | rte_mempool_put(message_pool, msg); 283 | } 284 | sleep(1); 285 | } 286 | } 287 | ``` 288 | 而接收端主要是采用`rte_ring_dequeue` 289 | ```c 290 | static int 291 | lcore_recv(__rte_unused void *arg) 292 | { 293 | unsigned lcore_id = rte_lcore_id(); 294 | printf("Starting core %u\n", lcore_id); 295 | while (!quit){ 296 | void *msg; 297 | if (rte_ring_dequeue(zartbot_ring, &msg) < 0){ 298 | usleep(5); 299 | continue; 300 | } 301 | printf("core %u: Received '%s'\n", lcore_id, (char *)msg); 302 | rte_mempool_put(message_pool, msg); 303 | } 304 | return 0; 305 | } 306 | ``` 307 | 308 | 309 | 310 | -------------------------------------------------------------------------------- /a1_setup_mlx5_sriov_env/centos.md: -------------------------------------------------------------------------------- 1 | 2 | ## 系统安装 3 | 4 | 系统采用Centos 8.3，服务器为Xeon 铂金版8259CL，网卡为Mellnox CX5双100G接口，两个接口回环. 5 | 6 | 在安装的时候请您勾选一下组件, 虚拟化的东西是后面在虚拟机里面跑DPDK需要用到的. 7 | 8 | - Server with GUI 9 | - Virtualization Client 10 | - Virtualization Hypervisor 11 | - Virtualization Tools 12 | 13 | 安装完成后使用`root`登录,然后备份原有的yum repository配置，并更换阿里云的源 14 | 15 | 16 | ```bash 17 | #备份原有的配置文件 18 | mkdir /etc/yum.repos.d/bak 19 | mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/bak/ 20 | #使用阿里云的源覆盖 21 | wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-8.repo 22 | yum install -y https://mirrors.aliyun.com/epel/epel-release-latest-8.noarch.rpm 23 | sed -i 's|^#baseurl=https://download.fedoraproject.org/pub|baseurl=https://mirrors.aliyun.com|' /etc/yum.repos.d/epel* 24 | sed -i 's|^metalink|#metalink|' /etc/yum.repos.d/epel* 25 | sudo dnf config-manager --set-enabled PowerTools 26 | 27 | yum makecache 28 | yum update 29 | ``` 30 | 31 | ### 安装开发软件 32 | 33 | 主要是用于后面安装mlx5 ofed和编译DPDK所需的工具，注意下面的一个都不能少 34 | 35 | > 特别是注意 libnl3-devel，如果缺失编译DPDK会出现找不到libibverbs很多函数的错误. 36 | ```bash 37 | yum groupinstall "Development tools" 38 | yum install gcc-gfortran kernel-modules-extra tcl tk tcsh terminator tmux kernel-rpm-macros elfutils-libelf-devel libnl3-devel meson createrepo numactl-devel 39 | pip3 install pyelftools 40 | ``` 41 | 42 | ### 启用iommu 43 | 44 | ```bash 45 | sudo vi /etc/default/grub 46 | 47 | //在 GRUB_CMDLINE_LINUX 行添加"intel_iommu=on iommu=pt" 48 | GRUB_CMDLINE_LINUX="crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet intel_iommu=on iommu=pt" 49 | //保存退出 50 | ``` 51 | 52 | 然后更新grub 53 | ```bash 54 | sudo grub2-mkconfig -o /boot/grub2/grub.cfg 55 | sudo grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg 56 | ``` 57 | ### 重启系统准备安装 58 | 59 | >此时必须要重启一次系统， 否则在ofed安装的时候会由于前面yum update了kernel报错. 60 | 61 |
62 |
63 |
64 | 65 | ## 安装MLX5-OFED 66 | 67 | 重启后继续使用`root`登录，在如下连接下载： 68 | 69 | ```mk 70 | https://www.mellanox.com/products/infiniband-drivers/linux/mlnx_ofed 71 | ``` 72 | 然后解压安装,注意安装时选择带上参数` --upstream-libs --dpdk --add-kernel-support ` 73 | 74 | ```bash 75 | tar vzxf MLNX_OFED_LINUX-5.3-1.0.0.1-rhel8.3-x86_64.tgz 76 | cd MLNX_OFED_LINUX-5.3-1.0.0.1-rhel8.3-x86_64/ 77 | 78 | ./mlnxofedinstall --upstream-libs --dpdk --add-kernel-support 79 | ``` 80 | 81 | 安装时间有点慢，等一下，完成后按照提示更新initramfs，然后重启 82 | 83 | ```bash 84 | dracut -f 85 | reboot 86 | ``` 87 | 88 | 89 |
90 |
91 |
92 | 93 | ## 编译DPDK 94 | 95 | 此时可以选择非root账户登录，然后编译DPDK， 首先需要检查openibd服务是否已经启动 96 | ```bash 97 | [zartbot@netdev ~]$ sudo systemctl status openibd 98 | ● openibd.service - openibd - configure Mellanox devices 99 | Loaded: loaded (/usr/lib/systemd/system/openibd.service; enabled; vendor preset: disabled) 100 | Active: active (exited) since Mon 2021-05-24 00:56:02 EDT; 1min 45s ago 101 | Docs: file:/etc/infiniband/openib.conf 102 | Process: 2714 ExecStart=/etc/init.d/openibd start bootid=e6af7c2ebc754526b55c8ef2c351d493 (code=exited, status=0/SUCCESS) 103 | Main PID: 2714 (code=exited, status=0/SUCCESS) 104 | Tasks: 0 (limit: 2465692) 105 | Memory: 24.1M 106 | CGroup: /system.slice/openibd.service 107 | 108 | May 24 00:56:01 netdev systemd[1]: Starting openibd - configure Mellanox devices... 109 | May 24 00:56:02 netdev openibd[2714]: [49B blob data] 110 | May 24 00:56:02 netdev systemd[1]: Started openibd - configure Mellanox devices. 111 | ``` 112 | 113 | 和ubuntu一类的系统不同，CentOS上需要添加/usr/local路径, 主要是`LD_LIBRARY_PATH` `PATH` 和 `PKG_CONFIG_PATH` 以及`sudo`的path 114 | ```bash 115 | sudo vi /etc/ld.so.conf.d/dpdk.conf 116 | 117 | >>添加如下path 118 | /usr/local/lib64 119 | >>退出 120 | 121 | sudo ldconfig 122 | 123 | vim ~/.bashrc 124 | >>添加如下path 125 | 126 | export PATH=/usr/local/bin:$PATH 127 | export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:${PKG_CONFIG_PATH} 128 | 129 | sudo vim /etc/sudoers 130 | 131 | >>将secure_path添加/usr/local/bin 132 | Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin 133 | ``` 134 | 然后解压dpdk，并编译安装 135 | 136 | ```bash 137 | tar xf dpdk-21.05.tar.xz 138 | 139 | cd dpdk-21.05 140 | meson build -D examples=all 141 | 142 | cd build 143 | ninja 144 | sudo ninja install 145 | sudo ldconfig 146 | ``` 147 | 148 | ## 检查device状态 149 | 150 | ```bash 151 | [zartbot@netdev build]$ sudo dpdk-devbind.py --status 152 | 153 | Network devices using kernel driver 154 | =================================== 155 | 0000:60:00.0 'Ethernet Connection X722 for 1GbE 37d1' if=enp96s0f0 drv=i40e unused= 156 | 0000:60:00.1 'Ethernet Connection X722 for 1GbE 37d1' if=enp96s0f1 drv=i40e unused= *Active* 157 | 0000:60:00.2 'Ethernet Connection X722 for 1GbE 37d1' if=enp96s0f2 drv=i40e unused= 158 | 0000:60:00.3 'Ethernet Connection X722 for 1GbE 37d1' if=enp96s0f3 drv=i40e unused= 159 | 0000:86:00.0 'MT28800 Family [ConnectX-5 Ex] 1019' if=ens17f0 drv=mlx5_core unused= 160 | 0000:86:00.1 'MT28800 Family [ConnectX-5 Ex] 1019' if=ens17f1 drv=mlx5_core unused= 161 | 162 | No 'Baseband' devices detected 163 | ============================== 164 | 165 | No 'Crypto' devices detected 166 | ============================ 167 | 168 | No 'Eventdev' devices detected 169 | ============================== 170 | 171 | No 'Mempool' devices detected 172 | ============================= 173 | 174 | No 'Compress' devices detected 175 | ============================== 176 | 177 | Misc (rawdev) devices using kernel driver 178 | ========================================= 179 | 0000:00:04.0 'Sky Lake-E CBDMA Registers 2021' drv=ioatdma unused= 180 | 0000:00:04.1 'Sky Lake-E CBDMA Registers 2021' drv=ioatdma unused= 181 | 0000:00:04.2 'Sky Lake-E CBDMA Registers 2021' drv=ioatdma unused= 182 | 0000:00:04.3 'Sky Lake-E CBDMA Registers 2021' drv=ioatdma unused= 183 | 0000:00:04.4 'Sky Lake-E CBDMA Registers 2021' drv=ioatdma unused= 184 | 0000:00:04.5 'Sky Lake-E CBDMA Registers 2021' drv=ioatdma unused= 185 | 0000:00:04.6 'Sky Lake-E CBDMA Registers 2021' drv=ioatdma unused= 186 | 0000:00:04.7 'Sky Lake-E CBDMA Registers 2021' drv=ioatdma unused= 187 | 188 | Other Misc (rawdev) devices 189 | =========================== 190 | 0000:80:04.0 'Sky Lake-E CBDMA Registers 2021' unused=ioatdma 191 | 0000:80:04.1 'Sky Lake-E CBDMA Registers 2021' unused=ioatdma 192 | 0000:80:04.2 'Sky Lake-E CBDMA Registers 2021' unused=ioatdma 193 | 0000:80:04.3 'Sky Lake-E CBDMA Registers 2021' unused=ioatdma 194 | 0000:80:04.4 'Sky Lake-E CBDMA Registers 2021' unused=ioatdma 195 | 0000:80:04.5 'Sky Lake-E CBDMA Registers 2021' unused=ioatdma 196 | 0000:80:04.6 'Sky Lake-E CBDMA Registers 2021' unused=ioatdma 197 | 0000:80:04.7 'Sky Lake-E CBDMA Registers 2021' unused=ioatdma 198 | 199 | No 'Regex' devices detected 200 | =========================== 201 | ``` 202 | 203 | ## 测试收发包 204 | 205 | >需要注意的是安装了ofed驱动和支持mlx5_pmd的网卡不需要bind，直接就可以使用，我们还是使用上次的框架，稍微改改收发包程序，来测一下pps 206 | 207 | 源代码可以在github/zartbot/learn_dpdk/a1_setup_mlx5_sriov_env找到 208 | 209 | 210 | 发包程序和上次的区别是直接一个for循环`rte_eth_tx_burst` 211 | ```c 212 | for(;;) { 213 | uint16_t nb_tx = rte_eth_tx_burst(0, 0, pkt, BURST_SIZE); 214 | } 215 | ``` 216 | 收包那段代码做了一个统计pps的功能 217 | 218 | ```c 219 | uint64_t freq = rte_get_tsc_hz() ; 220 | for (;;) 221 | { 222 | uint16_t nb_rx = rte_eth_rx_burst(1, 0, rx_pkt, BURST_SIZE); 223 | if (unlikely(nb_rx == 0)) 224 | { 225 | continue; 226 | } 227 | pkt_cnt += nb_rx; 228 | if (unlikely(rte_rdtsc() - now > freq)) { 229 | printf("PPS: %ld\n",pkt_cnt); 230 | pkt_cnt= 0; 231 | now = rte_rdtsc() ; 232 | } 233 | } 234 | ``` 235 | 236 | 测试之前需要注意PKG_CONFIG_PATH的定义,因为前面我们已经在bashrc中改过 237 | ```bash 238 | source ~/.bashrc 239 | make 240 | 241 | ``` 242 | 然后设置hugepage 243 | ```bash 244 | sudo dpdk-hugepages.py --setup 4G 245 | ``` 246 | 执行测试程序 247 | ```bash 248 | [zartbot@netdev test]$ sudo ./build/test 249 | EAL: Detected 96 lcore(s) 250 | EAL: Detected 2 NUMA nodes 251 | EAL: Detected shared linkage of DPDK 252 | EAL: Multi-process socket /var/run/dpdk/rte/mp_socket 253 | EAL: Selected IOVA mode 'VA' 254 | EAL: No available 1048576 kB hugepages reported 255 | EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0000:86:00.0 (socket 1) 256 | EAL: Probe PCI driver: mlx5_pci (15b3:1019) device: 0000:86:00.1 (socket 1) 257 | TELEMETRY: No legacy callbacks, legacy socket not created 258 | 259 | 260 | 261 | ***************************************** 262 | number of available port: 2 263 | 264 | 265 | initializing port 0... 266 | port[0] support RX cheksum offload. 267 | port[0] support TX mbuf fast free offload. 268 | port[0] support TX IPv4 checksum offload. 269 | port[0] support TX UDP checksum offload. 270 | port[0] support TX TCP checksum offload. 271 | Port[0] MAC: ec:0d:9a:c5:df:fc 272 | 273 | 274 | initializing port 1... 275 | port[1] support RX cheksum offload. 276 | port[1] support TX mbuf fast free offload. 277 | port[1] support TX IPv4 checksum offload. 278 | port[1] support TX UDP checksum offload. 279 | port[1] support TX TCP checksum offload. 280 | Port[1] MAC: ec:0d:9a:c5:df:fd 281 | PPS: 5751838 282 | PPS: 26864418 283 | PPS: 26815140 284 | PPS: 26820036 285 | PPS: 26829140 286 | ``` 287 | 288 |
289 |
290 |
291 | 292 | 293 | ## 配置VNC(可选) 294 | 295 | ```bash 296 | sudo dnf install tigervnc-server tigervnc-server-module 297 | sudo cp /usr/lib/systemd/system/vncserver@.service /etc/systemd/system/vncserver@.service 298 | sudo systemctl daemon-reload 299 | ``` 300 | 301 | 修改`vncserver.users` 例如5901给root，5902给zartbot， 302 | 303 | ```bash 304 | sudo vi /etc/tigervnc/vncserver.users 305 | >>在文件末尾添加 306 | :1=root 307 | ``` 308 | 配置分辨率和gnome 309 | 310 | ```bash 311 | sudo vi /etc/tigervnc/vncserver-config-defaults 312 | >>添加 313 | session=gnome 314 | geometry=2000x1200 315 | ``` 316 | 317 | 然后每个账号设置vnc-password 318 | 319 | ```bash 320 | [root@netdev build]# vncpasswd 321 | Password: 322 | Verify: 323 | Would you like to enter a view-only password (y/n)? n 324 | A view-only password is not used 325 | ``` 326 | 327 | 最后默认启动服务 328 | ```bash 329 | sudo systemctl enable vncserver@:1 330 | ``` 331 | 332 | 防火墙允许vnc服务 333 | ```bash 334 | sudo firewall-cmd --permanent --add-service vnc-server 335 | sudo firewall-cmd --reload 336 | ``` 337 | 此时还有可能vnc连不上，可以尝试重启一下就好 338 | 339 | 340 | 341 | 342 | 343 | 344 | 345 | 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | -------------------------------------------------------------------------------- /02_send_recv/test/main.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #define MAX_PORTS 16 16 | #define RX_RING_SIZE 1024 17 | #define TX_RING_SIZE 1024 18 | 19 | #define NUM_MBUFS 8191 20 | #define MBUF_CACHE_SIZE 250 21 | #define BURST_SIZE 32 22 | 23 | static const struct rte_eth_conf port_conf_default = { 24 | .rxmode = { 25 | .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 26 | }, 27 | .txmode = { 28 | .mq_mode = ETH_MQ_TX_NONE, 29 | } 30 | }; 31 | 32 | static inline int 33 | port_init(uint16_t port, struct rte_mempool *mbuf_pool) 34 | { 35 | struct rte_eth_conf port_conf = port_conf_default; 36 | const uint16_t rx_rings = 1, tx_rings = 1; 37 | uint16_t nb_rxd = RX_RING_SIZE; 38 | uint16_t nb_txd = TX_RING_SIZE; 39 | int retval; 40 | uint16_t q; 41 | struct rte_eth_dev_info dev_info; 42 | struct rte_eth_txconf txconf; 43 | 44 | if (!rte_eth_dev_is_valid_port(port)) 45 | return -1; 46 | 47 | retval = rte_eth_dev_info_get(port, &dev_info); 48 | if (retval != 0) 49 | { 50 | printf("Error during getting device (port %u) info: %s\n", 51 | port, strerror(-retval)); 52 | return retval; 53 | } 54 | printf("\n\ninitializing port %d...\n", port); 55 | 56 | if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_CHECKSUM) 57 | { 58 | printf("port[%u] support RX cheksum offload.\n", port); 59 | port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CHECKSUM; 60 | } 61 | 62 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 63 | { 64 | printf("port[%u] support TX mbuf fast free offload.\n", port); 65 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE; 66 | } 67 | 68 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MT_LOCKFREE) 69 | { 70 | printf("port[%u] support TX MT lock free offload.\n", port); 71 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MT_LOCKFREE; 72 | } 73 | 74 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 75 | { 76 | printf("port[%u] support TX IPv4 checksum offload.\n", port); 77 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 78 | } 79 | 80 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) 81 | { 82 | printf("port[%u] support TX UDP checksum offload.\n", port); 83 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_UDP_CKSUM; 84 | } 85 | 86 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM) 87 | { 88 | printf("port[%u] support TX TCP checksum offload.\n", port); 89 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_TCP_CKSUM; 90 | } 91 | 92 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM) 93 | { 94 | printf("port[%u] support TX SCTP checksum offload.\n", port); 95 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_SCTP_CKSUM; 96 | } 97 | 98 | /* Configure the Ethernet device. */ 99 | retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); 100 | if (retval != 0) 101 | return retval; 102 | 103 | retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); 104 | if (retval != 0) 105 | return retval; 106 | 107 | /* Allocate and set up 1 RX queue per Ethernet port. */ 108 | for (q = 0; q < rx_rings; q++) 109 | { 110 | retval = rte_eth_rx_queue_setup(port, q, nb_rxd, 111 | rte_eth_dev_socket_id(port), NULL, mbuf_pool); 112 | if (retval < 0) 113 | return retval; 114 | } 115 | 116 | txconf = dev_info.default_txconf; 117 | txconf.offloads = port_conf.txmode.offloads; 118 | /* Allocate and set up 1 TX queue per Ethernet port. */ 119 | for (q = 0; q < tx_rings; q++) 120 | { 121 | retval = rte_eth_tx_queue_setup(port, q, nb_txd, 122 | rte_eth_dev_socket_id(port), &txconf); 123 | if (retval < 0) 124 | return retval; 125 | } 126 | 127 | /* Start the Ethernet port. */ 128 | retval = rte_eth_dev_start(port); 129 | if (retval < 0) 130 | return retval; 131 | 132 | struct rte_eth_link link; 133 | do 134 | { 135 | retval = rte_eth_link_get_nowait(port, &link); 136 | if (retval < 0) 137 | { 138 | printf("Failed link get (port %u): %s\n", 139 | port, rte_strerror(-retval)); 140 | return retval; 141 | } 142 | else if (link.link_status) 143 | break; 144 | 145 | printf("Waiting for Link up on port %" PRIu16 "\n", port); 146 | sleep(1); 147 | } while (!link.link_status); 148 | 149 | /* Display the port MAC address. */ 150 | struct rte_ether_addr addr; 151 | retval = rte_eth_macaddr_get(port, &addr); 152 | if (retval != 0) 153 | return retval; 154 | 155 | printf("Port[%u] MAC: %02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 156 | ":%02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 "\n", 157 | port, 158 | addr.addr_bytes[0], addr.addr_bytes[1], 159 | addr.addr_bytes[2], addr.addr_bytes[3], 160 | addr.addr_bytes[4], addr.addr_bytes[5]); 161 | 162 | /* Enable RX in promiscuous mode for the Ethernet device. */ 163 | retval = rte_eth_promiscuous_enable(port); 164 | if (retval != 0) 165 | return retval; 166 | 167 | return 0; 168 | } 169 | 170 | rte_be32_t string_to_ip(char *s) 171 | { 172 | unsigned char a[4]; 173 | int rc = sscanf(s, "%hhd.%hhd.%hhd.%hhd", a + 0, a + 1, a + 2, a + 3); 174 | if (rc != 4) 175 | { 176 | fprintf(stderr, "bad source IP address format. Use like: 1.1.1.1\n"); 177 | exit(1); 178 | } 179 | return (rte_be32_t)(a[3]) << 24 | 180 | (rte_be32_t)(a[2]) << 16 | 181 | (rte_be32_t)(a[1]) << 8 | 182 | (rte_be32_t)(a[0]); 183 | } 184 | 185 | static int 186 | lcore_send(struct rte_mempool *mbuf_pool) { 187 | 188 | struct rte_ether_hdr *eth_hdr; 189 | struct rte_ipv4_hdr *ipv4_hdr; 190 | struct rte_udp_hdr *udp_hdr; 191 | 192 | //Defined header in UDP 193 | struct SRoU 194 | { 195 | uint8_t magic_num; 196 | uint8_t srou_length; 197 | uint8_t flags; 198 | uint8_t next_protcol; 199 | uint64_t pad; 200 | }; 201 | 202 | //init mac 203 | struct rte_ether_addr s_addr = {{0x14, 0x02, 0xEC, 0x89, 0x8D, 0x24}}; 204 | struct rte_ether_addr d_addr = {{0x3c, 0xfd, 0xfe, 0xa9, 0xa8, 0x89}}; 205 | 206 | //init IP header 207 | rte_be32_t s_ip_addr = string_to_ip("1.0.0.253"); 208 | rte_be32_t d_ip_addr = string_to_ip("1.0.0.1"); 209 | uint16_t ether_type = rte_cpu_to_be_16(0x0800); 210 | 211 | //init udp payload 212 | struct SRoU obj = { 213 | .magic_num = 1, 214 | .srou_length = 4, 215 | .flags = 0xFF, 216 | .next_protcol = 0, 217 | }; 218 | struct SRoU *msg; 219 | struct rte_mbuf *pkt[BURST_SIZE]; 220 | 221 | for (int i = 0; i < BURST_SIZE; i++) 222 | { 223 | pkt[i] = rte_pktmbuf_alloc(mbuf_pool); 224 | eth_hdr = rte_pktmbuf_mtod(pkt[i], struct rte_ether_hdr *); 225 | eth_hdr->d_addr = d_addr; 226 | struct rte_ether_addr s_addr = {{0x14, 0x02, 0xEC, 0x89, 0x8D, i}}; 227 | eth_hdr->s_addr = s_addr; 228 | eth_hdr->ether_type = ether_type; 229 | 230 | ipv4_hdr = rte_pktmbuf_mtod_offset(pkt[i], struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr)); 231 | ipv4_hdr->version_ihl = 0x45; 232 | ipv4_hdr->next_proto_id = 0x11; 233 | ipv4_hdr->src_addr = s_ip_addr; 234 | ipv4_hdr->dst_addr = d_ip_addr; 235 | ipv4_hdr->time_to_live = 0x40; 236 | 237 | udp_hdr = rte_pktmbuf_mtod_offset(pkt[i], struct rte_udp_hdr *, sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr)); 238 | udp_hdr->dgram_len = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr)); 239 | udp_hdr->src_port = rte_cpu_to_be_16(1234); 240 | udp_hdr->dst_port = rte_cpu_to_be_16(6666); 241 | ipv4_hdr->total_length = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr) + sizeof(struct rte_ipv4_hdr)); 242 | 243 | msg = (struct SRoU *)(rte_pktmbuf_mtod(pkt[i], char *) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr)); 244 | *msg = obj; 245 | int pkt_size = sizeof(struct SRoU) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr); 246 | 247 | pkt[i]->l2_len = sizeof(struct rte_ether_hdr); 248 | pkt[i]->l3_len = sizeof(struct rte_ipv4_hdr); 249 | pkt[i]->l4_len = sizeof(struct rte_udp_hdr); 250 | pkt[i]->ol_flags |= PKT_TX_IPV4 | PKT_TX_IP_CKSUM | PKT_TX_UDP_CKSUM; 251 | ipv4_hdr->hdr_checksum = 0; 252 | udp_hdr->dgram_cksum = rte_ipv4_phdr_cksum(ipv4_hdr, pkt[i]->ol_flags); 253 | pkt[i]->data_len = pkt_size; 254 | pkt[i]->pkt_len = pkt_size; 255 | } 256 | 257 | 258 | for(;;) { 259 | uint16_t nb_tx = rte_eth_tx_burst(0, 0, pkt, BURST_SIZE); 260 | printf("successful send %d pkts\n", nb_tx); 261 | sleep(2); 262 | 263 | } 264 | for (int i = 0; i < BURST_SIZE; i++) 265 | { 266 | rte_pktmbuf_free(pkt[i]); 267 | } 268 | 269 | } 270 | 271 | int main(int argc, char *argv[]) 272 | { 273 | 274 | struct rte_mempool *mbuf_pool; 275 | unsigned nb_ports; 276 | uint16_t portid; 277 | 278 | int ret = rte_eal_init(argc, argv); 279 | if (ret < 0) 280 | rte_exit(EXIT_FAILURE, "initlize fail!"); 281 | 282 | printf("\n\n\n*****************************************\n"); 283 | 284 | nb_ports = rte_eth_dev_count_avail(); 285 | printf("number of available port: %d\n", nb_ports); 286 | 287 | /* Creates a new mempool in memory to hold the mbufs. */ 288 | mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports, 289 | MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 290 | 291 | if (mbuf_pool == NULL) 292 | rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 293 | 294 | /* Initialize all ports. */ 295 | RTE_ETH_FOREACH_DEV(portid) 296 | if (port_init(portid, mbuf_pool) != 0) 297 | rte_exit(EXIT_FAILURE, "Cannot init port %" PRIu16 "\n", 298 | portid); 299 | 300 | /* start packet send function on lcore-1 */ 301 | rte_eal_remote_launch((lcore_function_t *)lcore_send,mbuf_pool,1); 302 | 303 | struct rte_mbuf *rx_pkt[BURST_SIZE]; 304 | for (int i = 0; i < BURST_SIZE; i++) 305 | { 306 | rx_pkt[i] = rte_pktmbuf_alloc(mbuf_pool); 307 | } 308 | 309 | for (;;) 310 | { 311 | uint16_t nb_rx = rte_eth_rx_burst(1, 0, rx_pkt, BURST_SIZE); 312 | if (nb_rx == 0) 313 | { 314 | continue; 315 | } 316 | struct rte_ether_hdr *eth_hdr; 317 | for (int i = 0; i < nb_rx; i++) 318 | { 319 | eth_hdr = rte_pktmbuf_mtod(rx_pkt[i], struct rte_ether_hdr *); 320 | printf("Recv Pkt[%d] from MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 321 | " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " \n",i, 322 | eth_hdr->s_addr.addr_bytes[0], eth_hdr->s_addr.addr_bytes[1], 323 | eth_hdr->s_addr.addr_bytes[2], eth_hdr->s_addr.addr_bytes[3], 324 | eth_hdr->s_addr.addr_bytes[4], eth_hdr->s_addr.addr_bytes[5]); 325 | rte_pktmbuf_free(rx_pkt[i]); 326 | } 327 | } 328 | return 0; 329 | } -------------------------------------------------------------------------------- /a1_setup_mlx5_sriov_env/test/main.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #define MAX_PORTS 16 16 | #define RX_RING_SIZE 1024 17 | #define TX_RING_SIZE 1024 18 | 19 | #define NUM_MBUFS 8191 20 | #define MBUF_CACHE_SIZE 250 21 | #define BURST_SIZE 32 22 | 23 | static int hwts_dynfield_offset = -1; 24 | 25 | static inline rte_mbuf_timestamp_t * 26 | hwts_field(struct rte_mbuf *mbuf) 27 | { 28 | return RTE_MBUF_DYNFIELD(mbuf, 29 | hwts_dynfield_offset, rte_mbuf_timestamp_t *); 30 | } 31 | 32 | typedef uint64_t tsc_t; 33 | static int tsc_dynfield_offset = -1; 34 | 35 | static inline tsc_t * 36 | tsc_field(struct rte_mbuf *mbuf) 37 | { 38 | return RTE_MBUF_DYNFIELD(mbuf, tsc_dynfield_offset, tsc_t *); 39 | } 40 | 41 | static uint16_t 42 | add_timestamps(uint16_t port __rte_unused, uint16_t qidx __rte_unused, 43 | struct rte_mbuf **pkts, uint16_t nb_pkts, 44 | uint16_t max_pkts __rte_unused, void *_ __rte_unused) 45 | { 46 | unsigned i; 47 | uint64_t now = rte_rdtsc(); 48 | 49 | for (i = 0; i < nb_pkts; i++) 50 | printf("packet[%d] sendtime: %ld\n",i,now); 51 | return nb_pkts; 52 | } 53 | 54 | static const struct rte_eth_conf port_conf_default = { 55 | .rxmode = { 56 | .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 57 | }, 58 | }; 59 | 60 | static inline int 61 | port_init(uint16_t port, struct rte_mempool *mbuf_pool) 62 | { 63 | struct rte_eth_conf port_conf = port_conf_default; 64 | const uint16_t rx_rings = 1, tx_rings = 1; 65 | uint16_t nb_rxd = RX_RING_SIZE; 66 | uint16_t nb_txd = TX_RING_SIZE; 67 | int retval; 68 | uint16_t q; 69 | struct rte_eth_dev_info dev_info; 70 | struct rte_eth_txconf txconf; 71 | 72 | if (!rte_eth_dev_is_valid_port(port)) 73 | return -1; 74 | 75 | retval = rte_eth_dev_info_get(port, &dev_info); 76 | if (retval != 0) 77 | { 78 | printf("Error during getting device (port %u) info: %s\n", 79 | port, strerror(-retval)); 80 | return retval; 81 | } 82 | printf("\n\ninitializing port %d...\n", port); 83 | 84 | 85 | if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_CHECKSUM) 86 | { 87 | printf("port[%u] support RX cheksum offload.\n", port); 88 | port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CHECKSUM; 89 | } 90 | 91 | 92 | if (!(dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TIMESTAMP)) { 93 | printf("\nERROR: Port %u does not support hardware timestamping\n" 94 | , port); 95 | return -1; 96 | } 97 | port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_TIMESTAMP; 98 | rte_mbuf_dyn_rx_timestamp_register(&hwts_dynfield_offset, NULL); 99 | if (hwts_dynfield_offset < 0) { 100 | printf("ERROR: Failed to register timestamp field\n"); 101 | return -rte_errno; 102 | } 103 | 104 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 105 | { 106 | printf("port[%u] support TX mbuf fast free offload.\n", port); 107 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE; 108 | } 109 | 110 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MT_LOCKFREE) 111 | { 112 | printf("port[%u] support TX MT lock free offload.\n", port); 113 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MT_LOCKFREE; 114 | } 115 | 116 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 117 | { 118 | printf("port[%u] support TX IPv4 checksum offload.\n", port); 119 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 120 | } 121 | 122 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) 123 | { 124 | printf("port[%u] support TX UDP checksum offload.\n", port); 125 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_UDP_CKSUM; 126 | } 127 | 128 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM) 129 | { 130 | printf("port[%u] support TX TCP checksum offload.\n", port); 131 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_TCP_CKSUM; 132 | } 133 | 134 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM) 135 | { 136 | printf("port[%u] support TX SCTP checksum offload.\n", port); 137 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_SCTP_CKSUM; 138 | } 139 | 140 | /* Configure the Ethernet device. */ 141 | retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); 142 | if (retval != 0) 143 | return retval; 144 | 145 | retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); 146 | if (retval != 0) 147 | return retval; 148 | 149 | /* Allocate and set up 1 RX queue per Ethernet port. */ 150 | for (q = 0; q < rx_rings; q++) 151 | { 152 | retval = rte_eth_rx_queue_setup(port, q, nb_rxd, 153 | rte_eth_dev_socket_id(port), NULL, mbuf_pool); 154 | if (retval < 0) 155 | return retval; 156 | } 157 | 158 | txconf = dev_info.default_txconf; 159 | txconf.offloads = port_conf.txmode.offloads; 160 | /* Allocate and set up 1 TX queue per Ethernet port. */ 161 | for (q = 0; q < tx_rings; q++) 162 | { 163 | retval = rte_eth_tx_queue_setup(port, q, nb_txd, 164 | rte_eth_dev_socket_id(port), &txconf); 165 | if (retval < 0) 166 | return retval; 167 | } 168 | 169 | /* Start the Ethernet port. */ 170 | retval = rte_eth_dev_start(port); 171 | if (retval < 0) 172 | return retval; 173 | 174 | struct rte_eth_link link; 175 | do 176 | { 177 | retval = rte_eth_link_get_nowait(port, &link); 178 | if (retval < 0) 179 | { 180 | printf("Failed link get (port %u): %s\n", 181 | port, rte_strerror(-retval)); 182 | return retval; 183 | } 184 | else if (link.link_status) 185 | break; 186 | 187 | printf("Waiting for Link up on port %" PRIu16 "\n", port); 188 | sleep(1); 189 | } while (!link.link_status); 190 | 191 | /* Display the port MAC address. */ 192 | struct rte_ether_addr addr; 193 | retval = rte_eth_macaddr_get(port, &addr); 194 | if (retval != 0) 195 | return retval; 196 | 197 | printf("Port[%u] MAC: %02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 198 | ":%02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 "\n", 199 | port, 200 | addr.addr_bytes[0], addr.addr_bytes[1], 201 | addr.addr_bytes[2], addr.addr_bytes[3], 202 | addr.addr_bytes[4], addr.addr_bytes[5]); 203 | 204 | /* Enable RX in promiscuous mode for the Ethernet device. */ 205 | retval = rte_eth_promiscuous_enable(port); 206 | if (retval != 0) 207 | return retval; 208 | 209 | //rte_eth_add_rx_callback(port, 0, add_timestamps, NULL); 210 | 211 | return 0; 212 | } 213 | 214 | rte_be32_t string_to_ip(char *s) 215 | { 216 | unsigned char a[4]; 217 | int rc = sscanf(s, "%hhd.%hhd.%hhd.%hhd", a + 0, a + 1, a + 2, a + 3); 218 | if (rc != 4) 219 | { 220 | fprintf(stderr, "bad source IP address format. Use like: 1.1.1.1\n"); 221 | exit(1); 222 | } 223 | return (rte_be32_t)(a[3]) << 24 | 224 | (rte_be32_t)(a[2]) << 16 | 225 | (rte_be32_t)(a[1]) << 8 | 226 | (rte_be32_t)(a[0]); 227 | } 228 | 229 | 230 | 231 | 232 | static int 233 | lcore_send(struct rte_mempool *mbuf_pool) { 234 | 235 | struct rte_ether_hdr *eth_hdr; 236 | struct rte_ipv4_hdr *ipv4_hdr; 237 | struct rte_udp_hdr *udp_hdr; 238 | 239 | //Defined header in UDP 240 | struct SRoU 241 | { 242 | uint8_t magic_num; 243 | uint8_t srou_length; 244 | uint8_t flags; 245 | uint8_t next_protcol; 246 | uint64_t pad; 247 | }; 248 | 249 | //init mac 250 | struct rte_ether_addr s_addr = {{0x14, 0x02, 0xEC, 0x89, 0x8D, 0x24}}; 251 | struct rte_ether_addr d_addr = {{0x3c, 0xfd, 0xfe, 0xa9, 0xa8, 0x89}}; 252 | 253 | //init IP header 254 | rte_be32_t s_ip_addr = string_to_ip("1.0.0.253"); 255 | rte_be32_t d_ip_addr = string_to_ip("1.0.0.1"); 256 | uint16_t ether_type = rte_cpu_to_be_16(0x0800); 257 | 258 | //init udp payload 259 | struct SRoU obj = { 260 | .magic_num = 1, 261 | .srou_length = 4, 262 | .flags = 0xFF, 263 | .next_protcol = 0, 264 | }; 265 | struct SRoU *msg; 266 | struct rte_mbuf *pkt[BURST_SIZE]; 267 | 268 | 269 | 270 | for(;;) { 271 | for (int i = 0; i < BURST_SIZE; i++) 272 | { 273 | pkt[i] = rte_pktmbuf_alloc(mbuf_pool); 274 | } 275 | 276 | for (int i = 0; i < BURST_SIZE; i++) 277 | { 278 | eth_hdr = rte_pktmbuf_mtod(pkt[i], struct rte_ether_hdr *); 279 | eth_hdr->d_addr = d_addr; 280 | struct rte_ether_addr s_addr = {{0x14, 0x02, 0xEC, 0x89, 0x8D, i}}; 281 | eth_hdr->s_addr = s_addr; 282 | eth_hdr->ether_type = ether_type; 283 | 284 | ipv4_hdr = rte_pktmbuf_mtod_offset(pkt[i], struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr)); 285 | ipv4_hdr->version_ihl = 0x45; 286 | ipv4_hdr->next_proto_id = 0x11; 287 | ipv4_hdr->src_addr = s_ip_addr; 288 | ipv4_hdr->dst_addr = d_ip_addr; 289 | ipv4_hdr->time_to_live = 0x40; 290 | 291 | udp_hdr = rte_pktmbuf_mtod_offset(pkt[i], struct rte_udp_hdr *, sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr)); 292 | udp_hdr->dgram_len = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr)); 293 | udp_hdr->src_port = rte_cpu_to_be_16(1234); 294 | udp_hdr->dst_port = rte_cpu_to_be_16(6666); 295 | ipv4_hdr->total_length = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr) + sizeof(struct rte_ipv4_hdr)); 296 | 297 | msg = (struct SRoU *)(rte_pktmbuf_mtod(pkt[i], char *) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr)); 298 | *msg = obj; 299 | int pkt_size = sizeof(struct SRoU) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr); 300 | 301 | pkt[i]->l2_len = sizeof(struct rte_ether_hdr); 302 | pkt[i]->l3_len = sizeof(struct rte_ipv4_hdr); 303 | pkt[i]->l4_len = sizeof(struct rte_udp_hdr); 304 | pkt[i]->ol_flags |= PKT_TX_IPV4 | PKT_TX_IP_CKSUM | PKT_TX_UDP_CKSUM; 305 | ipv4_hdr->hdr_checksum = 0; 306 | udp_hdr->dgram_cksum = rte_ipv4_phdr_cksum(ipv4_hdr, pkt[i]->ol_flags); 307 | pkt[i]->data_len = pkt_size; 308 | pkt[i]->pkt_len = pkt_size; 309 | } 310 | 311 | uint16_t nb_tx = rte_eth_tx_burst(0, 0, pkt, BURST_SIZE); 312 | for (int i = 0; i < BURST_SIZE; i++) 313 | { 314 | rte_pktmbuf_free(pkt[i]); 315 | } 316 | 317 | } 318 | 319 | } 320 | 321 | int main(int argc, char *argv[]) 322 | { 323 | 324 | struct rte_mempool *mbuf_pool; 325 | unsigned nb_ports; 326 | uint16_t portid; 327 | uint64_t now = rte_rdtsc(); 328 | uint64_t pkt_cnt = 0; 329 | 330 | int ret = rte_eal_init(argc, argv); 331 | if (ret < 0) 332 | rte_exit(EXIT_FAILURE, "initlize fail!"); 333 | 334 | printf("\n\n\n*****************************************\n"); 335 | 336 | nb_ports = rte_eth_dev_count_avail(); 337 | printf("number of available port: %d\n", nb_ports); 338 | 339 | /* Creates a new mempool in memory to hold the mbufs. */ 340 | mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports, 341 | MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 342 | 343 | if (mbuf_pool == NULL) 344 | rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 345 | 346 | /* Initialize all ports. */ 347 | RTE_ETH_FOREACH_DEV(portid) 348 | if (port_init(portid, mbuf_pool) != 0) 349 | rte_exit(EXIT_FAILURE, "Cannot init port %" PRIu16 "\n", 350 | portid); 351 | 352 | /* start packet send function on lcore-1 */ 353 | rte_eal_remote_launch((lcore_function_t *)lcore_send,mbuf_pool,1); 354 | 355 | struct rte_mbuf *rx_pkt[BURST_SIZE]; 356 | for (int i = 0; i < BURST_SIZE; i++) 357 | { 358 | rx_pkt[i] = rte_pktmbuf_alloc(mbuf_pool); 359 | } 360 | 361 | 362 | uint64_t freq = rte_get_tsc_hz() ; 363 | 364 | for (;;) 365 | { 366 | uint16_t nb_rx = rte_eth_rx_burst(1, 0, rx_pkt, BURST_SIZE); 367 | if (unlikely(nb_rx == 0)) 368 | { 369 | continue; 370 | } 371 | pkt_cnt += nb_rx; 372 | 373 | if (unlikely(rte_rdtsc() - now > freq)) { 374 | printf("PPS: %ld\n",pkt_cnt); 375 | pkt_cnt= 0; 376 | now = rte_rdtsc() ; 377 | } 378 | /*struct rte_ether_hdr *eth_hdr; 379 | for (int i = 0; i < nb_rx; i++) 380 | { 381 | eth_hdr = rte_pktmbuf_mtod(rx_pkt[i], struct rte_ether_hdr *); 382 | printf("Recv Pkt[%d] from MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 383 | " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " \n",i, 384 | eth_hdr->s_addr.addr_bytes[0], eth_hdr->s_addr.addr_bytes[1], 385 | eth_hdr->s_addr.addr_bytes[2], eth_hdr->s_addr.addr_bytes[3], 386 | eth_hdr->s_addr.addr_bytes[4], eth_hdr->s_addr.addr_bytes[5]); 387 | rte_pktmbuf_free(rx_pkt[i]); 388 | }*/ 389 | } 390 | return 0; 391 | } -------------------------------------------------------------------------------- /a1_setup_mlx5_sriov_env/test/main.c.bak: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #define MAX_PORTS 16 16 | #define RX_RING_SIZE 1024 17 | #define TX_RING_SIZE 1024 18 | 19 | #define NUM_MBUFS 8191 20 | #define MBUF_CACHE_SIZE 250 21 | #define BURST_SIZE 32 22 | 23 | static int hwts_dynfield_offset = -1; 24 | 25 | static inline rte_mbuf_timestamp_t * 26 | hwts_field(struct rte_mbuf *mbuf) 27 | { 28 | return RTE_MBUF_DYNFIELD(mbuf, 29 | hwts_dynfield_offset, rte_mbuf_timestamp_t *); 30 | } 31 | 32 | typedef uint64_t tsc_t; 33 | static int tsc_dynfield_offset = -1; 34 | 35 | static inline tsc_t * 36 | tsc_field(struct rte_mbuf *mbuf) 37 | { 38 | return RTE_MBUF_DYNFIELD(mbuf, tsc_dynfield_offset, tsc_t *); 39 | } 40 | 41 | static uint16_t 42 | add_timestamps(uint16_t port __rte_unused, uint16_t qidx __rte_unused, 43 | struct rte_mbuf **pkts, uint16_t nb_pkts, 44 | uint16_t max_pkts __rte_unused, void *_ __rte_unused) 45 | { 46 | unsigned i; 47 | uint64_t now = rte_rdtsc(); 48 | 49 | for (i = 0; i < nb_pkts; i++) 50 | printf("packet[%d] sendtime: %ld\n",i,now); 51 | return nb_pkts; 52 | } 53 | 54 | static const struct rte_eth_conf port_conf_default = { 55 | .rxmode = { 56 | .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 57 | }, 58 | }; 59 | 60 | static inline int 61 | port_init(uint16_t port, struct rte_mempool *mbuf_pool) 62 | { 63 | struct rte_eth_conf port_conf = port_conf_default; 64 | const uint16_t rx_rings = 1, tx_rings = 1; 65 | uint16_t nb_rxd = RX_RING_SIZE; 66 | uint16_t nb_txd = TX_RING_SIZE; 67 | int retval; 68 | uint16_t q; 69 | struct rte_eth_dev_info dev_info; 70 | struct rte_eth_txconf txconf; 71 | 72 | if (!rte_eth_dev_is_valid_port(port)) 73 | return -1; 74 | 75 | retval = rte_eth_dev_info_get(port, &dev_info); 76 | if (retval != 0) 77 | { 78 | printf("Error during getting device (port %u) info: %s\n", 79 | port, strerror(-retval)); 80 | return retval; 81 | } 82 | printf("\n\ninitializing port %d...\n", port); 83 | 84 | 85 | if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_CHECKSUM) 86 | { 87 | printf("port[%u] support RX cheksum offload.\n", port); 88 | port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CHECKSUM; 89 | } 90 | 91 | 92 | if (!(dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TIMESTAMP)) { 93 | printf("\nERROR: Port %u does not support hardware timestamping\n" 94 | , port); 95 | return -1; 96 | } 97 | port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_TIMESTAMP; 98 | rte_mbuf_dyn_rx_timestamp_register(&hwts_dynfield_offset, NULL); 99 | if (hwts_dynfield_offset < 0) { 100 | printf("ERROR: Failed to register timestamp field\n"); 101 | return -rte_errno; 102 | } 103 | 104 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 105 | { 106 | printf("port[%u] support TX mbuf fast free offload.\n", port); 107 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE; 108 | } 109 | 110 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MT_LOCKFREE) 111 | { 112 | printf("port[%u] support TX MT lock free offload.\n", port); 113 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MT_LOCKFREE; 114 | } 115 | 116 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) 117 | { 118 | printf("port[%u] support TX IPv4 checksum offload.\n", port); 119 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 120 | } 121 | 122 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) 123 | { 124 | printf("port[%u] support TX UDP checksum offload.\n", port); 125 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_UDP_CKSUM; 126 | } 127 | 128 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM) 129 | { 130 | printf("port[%u] support TX TCP checksum offload.\n", port); 131 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_TCP_CKSUM; 132 | } 133 | 134 | if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM) 135 | { 136 | printf("port[%u] support TX SCTP checksum offload.\n", port); 137 | port_conf.txmode.offloads |= DEV_TX_OFFLOAD_SCTP_CKSUM; 138 | } 139 | 140 | /* Configure the Ethernet device. */ 141 | retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); 142 | if (retval != 0) 143 | return retval; 144 | 145 | retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); 146 | if (retval != 0) 147 | return retval; 148 | 149 | /* Allocate and set up 1 RX queue per Ethernet port. */ 150 | for (q = 0; q < rx_rings; q++) 151 | { 152 | retval = rte_eth_rx_queue_setup(port, q, nb_rxd, 153 | rte_eth_dev_socket_id(port), NULL, mbuf_pool); 154 | if (retval < 0) 155 | return retval; 156 | } 157 | 158 | txconf = dev_info.default_txconf; 159 | txconf.offloads = port_conf.txmode.offloads; 160 | /* Allocate and set up 1 TX queue per Ethernet port. */ 161 | for (q = 0; q < tx_rings; q++) 162 | { 163 | retval = rte_eth_tx_queue_setup(port, q, nb_txd, 164 | rte_eth_dev_socket_id(port), &txconf); 165 | if (retval < 0) 166 | return retval; 167 | } 168 | 169 | /* Start the Ethernet port. */ 170 | retval = rte_eth_dev_start(port); 171 | if (retval < 0) 172 | return retval; 173 | 174 | struct rte_eth_link link; 175 | do 176 | { 177 | retval = rte_eth_link_get_nowait(port, &link); 178 | if (retval < 0) 179 | { 180 | printf("Failed link get (port %u): %s\n", 181 | port, rte_strerror(-retval)); 182 | return retval; 183 | } 184 | else if (link.link_status) 185 | break; 186 | 187 | printf("Waiting for Link up on port %" PRIu16 "\n", port); 188 | sleep(1); 189 | } while (!link.link_status); 190 | 191 | /* Display the port MAC address. */ 192 | struct rte_ether_addr addr; 193 | retval = rte_eth_macaddr_get(port, &addr); 194 | if (retval != 0) 195 | return retval; 196 | 197 | printf("Port[%u] MAC: %02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 198 | ":%02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 "\n", 199 | port, 200 | addr.addr_bytes[0], addr.addr_bytes[1], 201 | addr.addr_bytes[2], addr.addr_bytes[3], 202 | addr.addr_bytes[4], addr.addr_bytes[5]); 203 | 204 | /* Enable RX in promiscuous mode for the Ethernet device. */ 205 | retval = rte_eth_promiscuous_enable(port); 206 | if (retval != 0) 207 | return retval; 208 | 209 | //rte_eth_add_rx_callback(port, 0, add_timestamps, NULL); 210 | 211 | return 0; 212 | } 213 | 214 | rte_be32_t string_to_ip(char *s) 215 | { 216 | unsigned char a[4]; 217 | int rc = sscanf(s, "%hhd.%hhd.%hhd.%hhd", a + 0, a + 1, a + 2, a + 3); 218 | if (rc != 4) 219 | { 220 | fprintf(stderr, "bad source IP address format. Use like: 1.1.1.1\n"); 221 | exit(1); 222 | } 223 | return (rte_be32_t)(a[3]) << 24 | 224 | (rte_be32_t)(a[2]) << 16 | 225 | (rte_be32_t)(a[1]) << 8 | 226 | (rte_be32_t)(a[0]); 227 | } 228 | 229 | 230 | 231 | 232 | static int 233 | lcore_send(struct rte_mempool *mbuf_pool) { 234 | 235 | struct rte_ether_hdr *eth_hdr; 236 | struct rte_ipv4_hdr *ipv4_hdr; 237 | struct rte_udp_hdr *udp_hdr; 238 | 239 | //Defined header in UDP 240 | struct SRoU 241 | { 242 | uint8_t magic_num; 243 | uint8_t srou_length; 244 | uint8_t flags; 245 | uint8_t next_protcol; 246 | uint64_t pad; 247 | }; 248 | 249 | //init mac 250 | struct rte_ether_addr s_addr = {{0x14, 0x02, 0xEC, 0x89, 0x8D, 0x24}}; 251 | struct rte_ether_addr d_addr = {{0x3c, 0xfd, 0xfe, 0xa9, 0xa8, 0x89}}; 252 | 253 | //init IP header 254 | rte_be32_t s_ip_addr = string_to_ip("1.0.0.253"); 255 | rte_be32_t d_ip_addr = string_to_ip("1.0.0.1"); 256 | uint16_t ether_type = rte_cpu_to_be_16(0x0800); 257 | 258 | //init udp payload 259 | struct SRoU obj = { 260 | .magic_num = 1, 261 | .srou_length = 4, 262 | .flags = 0xFF, 263 | .next_protcol = 0, 264 | }; 265 | struct SRoU *msg; 266 | struct rte_mbuf *pkt[BURST_SIZE]; 267 | 268 | for (int i = 0; i < BURST_SIZE; i++) 269 | { 270 | pkt[i] = rte_pktmbuf_alloc(mbuf_pool); 271 | } 272 | 273 | for (int i = 0; i < BURST_SIZE; i++) 274 | { 275 | eth_hdr = rte_pktmbuf_mtod(pkt[i], struct rte_ether_hdr *); 276 | eth_hdr->d_addr = d_addr; 277 | struct rte_ether_addr s_addr = {{0x14, 0x02, 0xEC, 0x89, 0x8D, i}}; 278 | eth_hdr->s_addr = s_addr; 279 | eth_hdr->ether_type = ether_type; 280 | 281 | ipv4_hdr = rte_pktmbuf_mtod_offset(pkt[i], struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr)); 282 | ipv4_hdr->version_ihl = 0x45; 283 | ipv4_hdr->next_proto_id = 0x11; 284 | ipv4_hdr->src_addr = s_ip_addr; 285 | ipv4_hdr->dst_addr = d_ip_addr; 286 | ipv4_hdr->time_to_live = 0x40; 287 | 288 | udp_hdr = rte_pktmbuf_mtod_offset(pkt[i], struct rte_udp_hdr *, sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr)); 289 | udp_hdr->dgram_len = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr)); 290 | udp_hdr->src_port = rte_cpu_to_be_16(1234); 291 | udp_hdr->dst_port = rte_cpu_to_be_16(6666); 292 | ipv4_hdr->total_length = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr) + sizeof(struct rte_ipv4_hdr)); 293 | 294 | msg = (struct SRoU *)(rte_pktmbuf_mtod(pkt[i], char *) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr)); 295 | *msg = obj; 296 | int pkt_size = sizeof(struct SRoU) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr); 297 | 298 | pkt[i]->l2_len = sizeof(struct rte_ether_hdr); 299 | pkt[i]->l3_len = sizeof(struct rte_ipv4_hdr); 300 | pkt[i]->l4_len = sizeof(struct rte_udp_hdr); 301 | pkt[i]->ol_flags |= PKT_TX_IPV4 | PKT_TX_IP_CKSUM | PKT_TX_UDP_CKSUM; 302 | ipv4_hdr->hdr_checksum = 0; 303 | udp_hdr->dgram_cksum = rte_ipv4_phdr_cksum(ipv4_hdr, pkt[i]->ol_flags); 304 | pkt[i]->data_len = pkt_size; 305 | pkt[i]->pkt_len = pkt_size; 306 | } 307 | 308 | 309 | for(;;) { 310 | uint16_t nb_tx = rte_eth_tx_burst(0, 0, pkt, BURST_SIZE); 311 | //printf("successful send %d pkts\n", nb_tx); 312 | //sleep(2); 313 | 314 | } 315 | for (int i = 0; i < BURST_SIZE; i++) 316 | { 317 | rte_pktmbuf_free(pkt[i]); 318 | } 319 | 320 | } 321 | 322 | int main(int argc, char *argv[]) 323 | { 324 | 325 | struct rte_mempool *mbuf_pool; 326 | unsigned nb_ports; 327 | uint16_t portid; 328 | uint64_t now = rte_rdtsc(); 329 | uint64_t pkt_cnt = 0; 330 | 331 | int ret = rte_eal_init(argc, argv); 332 | if (ret < 0) 333 | rte_exit(EXIT_FAILURE, "initlize fail!"); 334 | 335 | printf("\n\n\n*****************************************\n"); 336 | 337 | nb_ports = rte_eth_dev_count_avail(); 338 | printf("number of available port: %d\n", nb_ports); 339 | 340 | /* Creates a new mempool in memory to hold the mbufs. */ 341 | mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports, 342 | MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 343 | 344 | if (mbuf_pool == NULL) 345 | rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 346 | 347 | /* Initialize all ports. */ 348 | RTE_ETH_FOREACH_DEV(portid) 349 | if (port_init(portid, mbuf_pool) != 0) 350 | rte_exit(EXIT_FAILURE, "Cannot init port %" PRIu16 "\n", 351 | portid); 352 | 353 | /* start packet send function on lcore-1 */ 354 | rte_eal_remote_launch((lcore_function_t *)lcore_send,mbuf_pool,1); 355 | 356 | struct rte_mbuf *rx_pkt[BURST_SIZE]; 357 | for (int i = 0; i < BURST_SIZE; i++) 358 | { 359 | rx_pkt[i] = rte_pktmbuf_alloc(mbuf_pool); 360 | } 361 | 362 | 363 | uint64_t freq = rte_get_tsc_hz() ; 364 | 365 | for (;;) 366 | { 367 | uint16_t nb_rx = rte_eth_rx_burst(1, 0, rx_pkt, BURST_SIZE); 368 | if (unlikely(nb_rx == 0)) 369 | { 370 | continue; 371 | } 372 | pkt_cnt += nb_rx; 373 | 374 | if (unlikely(rte_rdtsc() - now > freq)) { 375 | printf("PPS: %ld\n",pkt_cnt); 376 | pkt_cnt= 0; 377 | now = rte_rdtsc() ; 378 | } 379 | /*struct rte_ether_hdr *eth_hdr; 380 | for (int i = 0; i < nb_rx; i++) 381 | { 382 | eth_hdr = rte_pktmbuf_mtod(rx_pkt[i], struct rte_ether_hdr *); 383 | printf("Recv Pkt[%d] from MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 384 | " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " \n",i, 385 | eth_hdr->s_addr.addr_bytes[0], eth_hdr->s_addr.addr_bytes[1], 386 | eth_hdr->s_addr.addr_bytes[2], eth_hdr->s_addr.addr_bytes[3], 387 | eth_hdr->s_addr.addr_bytes[4], eth_hdr->s_addr.addr_bytes[5]); 388 | rte_pktmbuf_free(rx_pkt[i]); 389 | }*/ 390 | } 391 | return 0; 392 | } --------------------------------------------------------------------------------