├── mtcp └── src │ ├── include │ ├── cpu.h │ ├── nic_control.h │ ├── eth_in.h │ ├── rss.h │ ├── pipe.h │ ├── arp.h │ ├── ip_out.h │ ├── memory_mgt.h │ ├── splice_pool.h │ ├── tcp_util.h │ ├── tcp_sb_queue.h │ ├── tcp_rb_frag_queue.h │ ├── logger.h │ ├── config.h │ ├── schedule.h │ ├── eth_out.h │ ├── timer.h │ ├── eventpoll.h │ ├── fhash.h │ ├── tcp_send_buffer.h │ ├── stat.h │ ├── socket.h │ ├── mtcp_epoll.h │ ├── icmp.h │ ├── tcp_out.h │ ├── ip_in.h │ ├── tcp_stream_queue.h │ ├── tcp_ring_buffer.h │ ├── addr_pool.h │ ├── tcp_in.h │ ├── mtcp_api.h │ ├── io_module.h │ └── debug.h │ ├── eth_in.c │ ├── cpu.c │ ├── intel_lthread │ ├── arch │ │ └── x86 │ │ │ ├── ctx.h │ │ │ └── ctx.c │ ├── lthread_mutex.h │ ├── lthread_tls.h │ ├── lthread_timer.h │ ├── lthread_cond.h │ ├── lthread.h │ ├── lthread_diag.h │ └── lthread_objcache.h │ ├── ip_in.c │ ├── nic_control.c │ ├── rss.c │ ├── socket.c │ ├── schedule.c │ ├── Makefile.in │ ├── tcp_sb_queue.c │ ├── tcp_rb_frag_queue.c │ ├── ip_out.c │ └── icmp.c ├── bootstrap.sh ├── nic ├── splice │ ├── load_p4 │ ├── build_p4 │ ├── config │ │ └── default.p4cfg │ └── src │ │ ├── include │ │ ├── rss.h │ │ ├── common.h │ │ └── packet_management.h │ │ └── splice.p4 ├── setup_teardown │ ├── build_p4 │ ├── load_p4 │ ├── config │ │ └── default.p4cfg │ └── src │ │ ├── include │ │ ├── nfp_device.h │ │ ├── rss.h │ │ ├── nfp_util.h │ │ ├── common.h │ │ └── conn_table.h │ │ └── dataplane.p4 └── scripts │ └── nfp-sdk6-rte.service ├── apps ├── epproxy │ ├── src │ │ ├── config.h │ │ ├── balance.h │ │ ├── http_parser.h │ │ ├── persist.h │ │ ├── http_stream.h │ │ ├── balance.c │ │ ├── util.h │ │ ├── hash.c │ │ ├── backend_pool.h │ │ ├── hash.h │ │ └── epproxy.h │ ├── config │ │ ├── sample_route.conf │ │ ├── sample_arp.conf │ │ ├── sample_epproxy.yaml │ │ └── sample_mtcp.conf │ ├── lib │ │ └── install_yaml.sh │ └── Makefile.in └── example │ ├── config │ ├── sample_route.conf │ ├── sample_arp.conf │ └── sample_mtcp.conf │ ├── util.h │ └── Makefile.in ├── load_nic.sh ├── init_nic.sh ├── LICENSE ├── Makefile.am └── configure.ac /mtcp/src/include/cpu.h: -------------------------------------------------------------------------------- 1 | #ifndef __CPU_H_ 2 | #define __CPU_H_ 3 | 4 | int GetNumCPUs(); 5 | 6 | #endif /* __CPU_H_ */ 7 | -------------------------------------------------------------------------------- /bootstrap.sh: -------------------------------------------------------------------------------- 1 | aclocal \ 2 | && libtoolize --copy \ 3 | && autoheader \ 4 | && automake --gnu --add-missing --copy \ 5 | && autoreconf --force 6 | -------------------------------------------------------------------------------- /nic/splice/load_p4: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | /opt/netronome/p4/bin/rtecli -n design-load -f src/build/splice.nffw -p src/build/pif_design.json -c config/default.p4cfg 3 | -------------------------------------------------------------------------------- /nic/splice/build_p4: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | rm -rf build 3 | /opt/netronome/p4/bin/nfp4build -o src/build/splice.nffw -l starfighter1 -4 src/splice.p4 -c src/splice.c -I src/include 4 | rm Makefile-nfp4build 5 | -------------------------------------------------------------------------------- /nic/setup_teardown/build_p4: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | rm -rf src/build 3 | /opt/netronome/p4/bin/nfp4build -o src/build/dataplane.nffw -l starfighter1 -4 src/dataplane.p4 -c src/dataplane.c -I src/include -DPIF_PLUGIN_INIT --no-debug-info 4 | rm Makefile-nfp4build 5 | -------------------------------------------------------------------------------- /mtcp/src/include/nic_control.h: -------------------------------------------------------------------------------- 1 | #ifndef __NIC_CONTROL_H_ 2 | #define __NIC_CONTROL_H_ 3 | 4 | int 5 | ProcessSpliceFinishPacket(mtcp_manager_t mtcp, uint32_t cur_ts, 6 | const int ifidx, unsigned char* pkt_data, int len); 7 | 8 | #endif /* __NIC_CONTROL_H_ */ 9 | -------------------------------------------------------------------------------- /mtcp/src/include/eth_in.h: -------------------------------------------------------------------------------- 1 | #ifndef __ETH_IN_H_ 2 | #define __ETH_IN_H_ 3 | 4 | #include "mtcp.h" 5 | 6 | int 7 | ProcessPacket(mtcp_manager_t mtcp, const int ifidx, 8 | uint32_t cur_ts, unsigned char *pkt_data, int len); 9 | 10 | #endif /* __ETH_IN_H_ */ 11 | -------------------------------------------------------------------------------- /nic/setup_teardown/load_p4: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | /opt/netronome/p4/bin/rtecli -n -p 20206 design-load -f src/build/dataplane.nffw -p src/build/pif_design.json -c config/default.p4cfg 3 | #/opt/netronome/p4/bin/rtecli -n -p 20207 design-load -f src/build/dataplane.nffw -p src/build/pif_design.json -c config/default.p4cfg 4 | -------------------------------------------------------------------------------- /mtcp/src/include/rss.h: -------------------------------------------------------------------------------- 1 | #ifndef __RSS_H_ 2 | #define __RSS_H_ 3 | 4 | #include 5 | 6 | /* sip, dip, sp, dp: in network byte order */ 7 | int GetRSSCPUCore(in_addr_t sip, in_addr_t dip, 8 | in_port_t sp, in_port_t dp, int num_queues, 9 | uint8_t endian_check); 10 | 11 | #endif /* __RSS_H_ */ 12 | -------------------------------------------------------------------------------- /apps/epproxy/src/config.h: -------------------------------------------------------------------------------- 1 | #ifndef __CONFIG_H_ 2 | #define __CONFIG_H_ 3 | /*---------------------------------------------------------------------------*/ 4 | struct proxy_context* 5 | LoadConfigData(const char *fname); 6 | /*---------------------------------------------------------------------------*/ 7 | #endif /* __CONFIG_H_ */ 8 | -------------------------------------------------------------------------------- /apps/epproxy/config/sample_route.conf: -------------------------------------------------------------------------------- 1 | # This file is routing table example of our testbed machine 2 | # Copy this file to route.conf and give appropriate routes 3 | # Please save this file as config/route.conf. Put the config/ 4 | # directory in the same directory where the binary lies. 5 | # 6 | # (Destination address)/(Prefix) (Device name) 7 | 8 | ROUTES 2 9 | 10.0.0.1/24 dpdk0 10 | 10.0.1.1/24 dpdk1 11 | -------------------------------------------------------------------------------- /apps/example/config/sample_route.conf: -------------------------------------------------------------------------------- 1 | # This file is routing table example of our testbed machine 2 | # Copy this file to route.conf and give appropriate routes 3 | # Please save this file as config/route.conf. Put the config/ 4 | # directory in the same directory where the binary lies. 5 | # 6 | # (Destination address)/(Prefix) (Device name) 7 | 8 | ROUTES 2 9 | 10.0.0.1/24 dpdk0 10 | 10.0.1.1/24 dpdk1 11 | -------------------------------------------------------------------------------- /apps/epproxy/config/sample_arp.conf: -------------------------------------------------------------------------------- 1 | # This file is to configure static arp tables. 2 | # Rename this file to arp.conf and set the appropriate values. 3 | # Please save this file as config/arp.conf. Put the config/ 4 | # directory in the same directory where the binary lies. 5 | # 6 | # (Destination IP address/IP_prefix) (Destination MAC address) 7 | 8 | ARP_ENTRY 2 9 | 10.0.0.1/32 00:00:00:00:00:01 10 | 10.0.1.1/32 00:00:00:00:00:02 -------------------------------------------------------------------------------- /apps/epproxy/lib/install_yaml.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | LIBDIR="lib" 4 | LIBYAML_VERSION="yaml-0.1.7" 5 | LIBYAML_HOME=${LIBYAML_VERSION} 6 | 7 | cd ${LIBDIR} 8 | 9 | # get libyaml 0.1.7 10 | wget http://pyyaml.org/download/libyaml/${LIBYAML_VERSION}.tar.gz 11 | 12 | # build libyaml 13 | pushd . 14 | tar xzf ${LIBYAML_VERSION}.tar.gz && cd ${LIBYAML_HOME} 15 | ./configure && cmake ./ && make 16 | cd ${PWD}/${DEPDIR} 17 | popd 18 | -------------------------------------------------------------------------------- /apps/example/config/sample_arp.conf: -------------------------------------------------------------------------------- 1 | # This file is to configure static arp tables. 2 | # Rename this file to arp.conf and set the appropriate values. 3 | # Please save this file as config/arp.conf. Put the config/ 4 | # directory in the same directory where the binary lies. 5 | # 6 | # (Destination IP address/IP_prefix) (Destination MAC address) 7 | 8 | ARP_ENTRY 2 9 | 10.0.0.1/32 00:00:00:00:00:01 10 | 10.0.1.1/32 00:00:00:00:00:02 -------------------------------------------------------------------------------- /mtcp/src/include/pipe.h: -------------------------------------------------------------------------------- 1 | #ifndef __MTCP_PIPE_H_ 2 | #define __MTCP_PIPE_H_ 3 | 4 | #include 5 | 6 | int 7 | PipeRead(mctx_t mctx, int pipeid, char *buf, int len); 8 | 9 | int 10 | PipeWrite(mctx_t mctx, int pipeid, const char *buf, int len); 11 | 12 | int 13 | RaisePendingPipeEvents(mctx_t mctx, int epid, int pipeid); 14 | 15 | int 16 | PipeClose(mctx_t mctx, int pipeid); 17 | 18 | #endif /* __MTCP_PIPE_H_ */ 19 | -------------------------------------------------------------------------------- /apps/epproxy/src/balance.h: -------------------------------------------------------------------------------- 1 | #ifndef HTTP_BACKEND_H 2 | #define HTTP_BACKEND_H 3 | #include "http_stream.h" 4 | #include "persist.h" 5 | #include "config.h" 6 | /*----------------------------------------------------------------------------*/ 7 | void 8 | DecideBackendServer(struct sticky_table *sticky_map, 9 | int* rr_count, struct http_stream *hs); 10 | /*----------------------------------------------------------------------------*/ 11 | #endif 12 | -------------------------------------------------------------------------------- /load_nic.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | if [ "$1" == "--server" ]; then 4 | if [ ! -f nic/setup_teardown/src/build/dataplane.nffw ]; then 5 | echo "Cannot find nic/setup_teardown/src/build/dataplane.nffw.. please run init_nic.sh first" 6 | exit 1 7 | fi 8 | cd nic/setup_teardown; ./load_p4; cd ../.. 9 | 10 | elif [ "$1" == "--proxy" ]; then 11 | if [ ! -f nic/splice/src/build/splice.nffw ]; then 12 | echo "Cannot find nic/splice/src/build/splice.nffw.. please run init_nic.sh first" 13 | exit 1 14 | fi 15 | cd nic/splice; ./load_p4; cd ../.. 16 | 17 | else 18 | echo "Usage: ./load_nic.sh [--server|--proxy]" 19 | 20 | fi 21 | -------------------------------------------------------------------------------- /mtcp/src/include/arp.h: -------------------------------------------------------------------------------- 1 | #ifndef __ARP_H_ 2 | #define __ARP_H_ 3 | 4 | #define MAX_ARPENTRY 1024 5 | 6 | int 7 | InitARPTable(); 8 | 9 | unsigned char * 10 | GetHWaddr(uint32_t ip); 11 | 12 | unsigned char * 13 | GetDestinationHWaddr(uint32_t dip, uint8_t is_gateway); 14 | 15 | void 16 | RequestARP(mtcp_manager_t mtcp, uint32_t ip, int nif, uint32_t cur_ts); 17 | 18 | int 19 | ProcessARPPacket(mtcp_manager_t mtcp, uint32_t cur_ts, 20 | const int ifidx, unsigned char* pkt_data, int len); 21 | 22 | void 23 | ARPTimer(mtcp_manager_t mtcp, uint32_t cur_ts); 24 | 25 | void 26 | PrintARPTable(); 27 | 28 | #endif /* __ARP_H_ */ 29 | -------------------------------------------------------------------------------- /mtcp/src/include/ip_out.h: -------------------------------------------------------------------------------- 1 | #ifndef __IP_OUT_H_ 2 | #define __IP_OUT_H_ 3 | 4 | #include 5 | #include "tcp_stream.h" 6 | 7 | extern inline int 8 | GetOutputInterface(uint32_t daddr, uint8_t* is_external); 9 | 10 | void 11 | ForwardIPv4Packet(mtcp_manager_t mtcp, int nif_in, char *buf, int len); 12 | 13 | uint8_t * 14 | IPOutputStandalone(struct mtcp_manager *mtcp, uint16_t eth_type, uint8_t protocol, 15 | uint16_t ip_id, uint32_t saddr, uint32_t daddr, uint16_t tcplen); 16 | 17 | uint8_t * 18 | IPOutput(struct mtcp_manager *mtcp, tcp_stream *stream, uint16_t tcplen, 19 | struct mtcp_offload_meta* offload_meta); 20 | 21 | #endif /* __IP_OUT_H_ */ 22 | -------------------------------------------------------------------------------- /mtcp/src/include/memory_mgt.h: -------------------------------------------------------------------------------- 1 | #ifndef __MEMORY_MGT_H_ 2 | #define __MEMORY_MGT_H_ 3 | 4 | struct mem_pool; 5 | typedef struct mem_pool* mem_pool_t; 6 | 7 | /* create a memory pool with a chunk size and total size 8 | an return the pointer to the memory pool */ 9 | mem_pool_t MPCreate(int chunk_size, size_t total_size, int is_hugepage); 10 | 11 | /* allocate one chunk */ 12 | void *MPAllocateChunk(mem_pool_t mp); 13 | 14 | /* free one chunk */ 15 | void MPFreeChunk(mem_pool_t mp, void *p); 16 | 17 | /* destroy the memory pool */ 18 | void MPDestroy(mem_pool_t mp); 19 | 20 | /* return the number of free chunks */ 21 | int MPGetFreeChunks(mem_pool_t mp); 22 | 23 | #endif /* __MEMORY_MGT_H_ */ 24 | -------------------------------------------------------------------------------- /mtcp/src/include/splice_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef __SPLICE_POOL_H_ 2 | #define __SPLICE_POOL_H_ 3 | 4 | #include 5 | #include 6 | 7 | int GetNumSplice(addr_pool_t ap); 8 | /*----------------------------------------------------------------------------*/ 9 | int 10 | MoveAddressToSplice(addr_pool_t ap, const struct sockaddr_in *addr); 11 | /*----------------------------------------------------------------------------*/ 12 | int 13 | FreeSpliceAddress(addr_pool_t ap, const struct sockaddr_in *addr); 14 | /*----------------------------------------------------------------------------*/ 15 | int 16 | SearchSpliceAddress(addr_pool_t ap, const struct sockaddr_in *addr); 17 | 18 | #endif /* __SPLICE_POOL_H_ */ 19 | -------------------------------------------------------------------------------- /mtcp/src/include/tcp_util.h: -------------------------------------------------------------------------------- 1 | #ifndef __TCP_UTIL_H_ 2 | #define __TCP_UTIL_H_ 3 | 4 | #include "mtcp.h" 5 | #include "tcp_stream.h" 6 | 7 | void ParseTCPOptions(tcp_stream *cur_stream, 8 | uint32_t cur_ts, uint8_t *tcpopt, int len); 9 | 10 | extern inline int 11 | ParseTCPTimestamp(tcp_stream *cur_stream, 12 | struct tcp_timestamp *ts, uint8_t *tcpopt, int len); 13 | 14 | #if TCP_OPT_SACK_ENABLED 15 | void 16 | ParseSACKOption(tcp_stream *cur_stream, 17 | uint32_t ack_seq, uint8_t *tcpopt, int len); 18 | #endif 19 | 20 | uint16_t 21 | TCPCalcChecksum(uint16_t *buf, uint16_t len, uint32_t saddr, uint32_t daddr); 22 | 23 | void 24 | PrintTCPOptions(uint8_t *tcpopt, int len); 25 | 26 | #endif /* __TCP_UTIL_H_ */ 27 | -------------------------------------------------------------------------------- /nic/setup_teardown/config/default.p4cfg: -------------------------------------------------------------------------------- 1 | { 2 | "tables": { 3 | "fwd_tbl": { 4 | "rules": [ 5 | { 6 | "action": { 7 | "type": "from_wire" 8 | }, 9 | "name": "from_wire", 10 | "match": { 11 | "standard_metadata.ingress_port": { 12 | "value": "p0" 13 | } 14 | } 15 | } 16 | ], 17 | "default_rule": { 18 | "action": { 19 | "type": "from_host" 20 | }, 21 | "name": "from_host" 22 | } 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /mtcp/src/include/tcp_sb_queue.h: -------------------------------------------------------------------------------- 1 | #ifndef __TCP_SB_QUEUE_ 2 | #define __TCP_SB_QUEUE_ 3 | 4 | #include "tcp_send_buffer.h" 5 | 6 | /*---------------------------------------------------------------------------*/ 7 | typedef struct sb_queue* sb_queue_t; 8 | /*---------------------------------------------------------------------------*/ 9 | sb_queue_t 10 | CreateSBQueue(int capacity); 11 | /*---------------------------------------------------------------------------*/ 12 | void 13 | DestroySBQueue(sb_queue_t sq); 14 | /*---------------------------------------------------------------------------*/ 15 | int 16 | SBEnqueue(sb_queue_t sq, struct tcp_send_buffer *buf); 17 | /*---------------------------------------------------------------------------*/ 18 | struct tcp_send_buffer * 19 | SBDequeue(sb_queue_t sq); 20 | /*---------------------------------------------------------------------------*/ 21 | 22 | #endif /* __TCP_SB_QUEUE_ */ 23 | -------------------------------------------------------------------------------- /mtcp/src/include/tcp_rb_frag_queue.h: -------------------------------------------------------------------------------- 1 | #ifndef __TCP_RB_FRAG_QUEUE_ 2 | #define __TCP_RB_FRAG_QUEUE_ 3 | 4 | #include "tcp_ring_buffer.h" 5 | 6 | /*---------------------------------------------------------------------------*/ 7 | typedef struct rb_frag_queue* rb_frag_queue_t; 8 | /*---------------------------------------------------------------------------*/ 9 | rb_frag_queue_t 10 | CreateRBFragQueue(int capacity); 11 | /*---------------------------------------------------------------------------*/ 12 | void 13 | DestroyRBFragQueue(rb_frag_queue_t rb_fragq); 14 | /*---------------------------------------------------------------------------*/ 15 | int 16 | RBFragEnqueue(rb_frag_queue_t rb_fragq, struct fragment_ctx *frag); 17 | /*---------------------------------------------------------------------------*/ 18 | struct fragment_ctx * 19 | RBFragDequeue(rb_frag_queue_t rb_fragq); 20 | /*---------------------------------------------------------------------------*/ 21 | 22 | #endif /* __TCP_RB_FRAG_QUEUE_ */ 23 | -------------------------------------------------------------------------------- /mtcp/src/include/logger.h: -------------------------------------------------------------------------------- 1 | #ifndef __LOGGER_H_ 2 | #define __LOGGER_H_ 3 | 4 | #include 5 | 6 | #define LOG_BUFF_SIZE (256*1024) 7 | #define NUM_LOG_BUFF (100) 8 | 9 | enum { 10 | IDLE_LOGT, 11 | ACTIVE_LOGT 12 | } log_thread_state; 13 | 14 | typedef struct log_buff 15 | { 16 | int tid; 17 | FILE* fid; 18 | int buff_len; 19 | char buff[LOG_BUFF_SIZE]; 20 | TAILQ_ENTRY(log_buff) buff_link; 21 | } log_buff; 22 | 23 | typedef struct log_thread_context { 24 | pthread_t thread; 25 | int cpu; 26 | int done; 27 | int sp_fd; 28 | int pair_sp_fd; 29 | int free_buff_cnt; 30 | int job_buff_cnt; 31 | 32 | uint8_t state; 33 | 34 | pthread_mutex_t mutex; 35 | pthread_mutex_t free_mutex; 36 | 37 | TAILQ_HEAD(, log_buff) working_queue; 38 | TAILQ_HEAD(, log_buff) free_queue; 39 | 40 | } log_thread_context; 41 | 42 | log_buff* DequeueFreeBuffer (log_thread_context *ctx); 43 | void EnqueueJobBuffer(log_thread_context *ctx, log_buff* working_bp); 44 | void InitLogThreadContext (log_thread_context *ctx, int cpu); 45 | void *ThreadLogMain(void* arg); 46 | #endif /* __LOGGER_H_ */ 47 | -------------------------------------------------------------------------------- /mtcp/src/include/config.h: -------------------------------------------------------------------------------- 1 | #ifndef __CONFIG_H_ 2 | #define __CONFIG_H_ 3 | 4 | #ifndef DISABLE_PSIO 5 | #include "ps.h" 6 | #endif 7 | 8 | #if defined(DISABLE_PSIO) && !defined(MAX_DEVICES) 9 | #define MAX_DEVICES 16 10 | #endif 11 | 12 | int num_cpus; 13 | int num_queues; 14 | int num_devices; 15 | 16 | int num_devices_attached; 17 | int devices_attached[MAX_DEVICES]; 18 | 19 | int 20 | LoadConfiguration(const char *fname); 21 | 22 | /* set configurations from the setted 23 | interface information */ 24 | int 25 | SetInterfaceInfo(); 26 | 27 | /* set configurations from the files */ 28 | int 29 | SetRoutingTable(); 30 | 31 | int 32 | LoadARPTable(); 33 | 34 | /* print setted configuration */ 35 | void 36 | PrintConfiguration(); 37 | 38 | void 39 | PrintInterfaceInfo(); 40 | 41 | void 42 | PrintRoutingTable(); 43 | 44 | /* set socket modes */ 45 | int 46 | SetSocketMode(int8_t socket_mode); 47 | 48 | /* fetch mask from prefix */ 49 | uint32_t 50 | MaskFromPrefix(int prefix); 51 | 52 | void 53 | ParseMACAddress(unsigned char *haddr, char *haddr_str); 54 | 55 | int 56 | ParseIPAddress(uint32_t *ip_addr, char *ip_str); 57 | 58 | #endif /* __CONFIG_H_ */ 59 | -------------------------------------------------------------------------------- /mtcp/src/include/schedule.h: -------------------------------------------------------------------------------- 1 | #ifndef _SCHEDULE_H_ 2 | #define _SCHEDULE_H_ 3 | 4 | #include 5 | #include "mtcp.h" 6 | 7 | /* global constants */ 8 | #define TIMER_SIGNAL SIGRTMIN 9 | #define YIELD_REASON_TIMER 0x1 10 | #define YIELD_REASON_EPOLL 0x2 11 | #define YIELD_REASON_BLOCK 0x4 12 | 13 | #ifdef ENABLE_APP_INTERRUPT 14 | 15 | /* initialize and delete per-thread timer */ 16 | int 17 | InitTimer(mtcp_thread_context_t mtcp); 18 | 19 | int 20 | DeleteTimer(mtcp_thread_context_t mtcp); 21 | 22 | /* setup timer interrupt */ 23 | int 24 | SetupTimerInterrupt(mtcp_thread_context_t mtcp); 25 | 26 | void 27 | HandleTimerInterrupt(int sig, siginfo_t *si, void *uc); 28 | 29 | /* enable and disable timer interrupt */ 30 | int 31 | EnableTimerInterrupt(mtcp_thread_context_t mtcp); 32 | 33 | int 34 | DisableTimerInterrupt(mtcp_thread_context_t mtcp); 35 | 36 | #endif /* ENABLE_APP_INTERRUPT */ 37 | 38 | /* yield between stack context and app context */ 39 | inline void 40 | YieldToStack(mtcp_thread_context_t mtcp, int reason); 41 | 42 | inline void 43 | YieldToApp(mtcp_thread_context_t mtcp, int interrupt); 44 | 45 | #endif /* _SCHEDULE_H_ */ 46 | -------------------------------------------------------------------------------- /apps/epproxy/config/sample_epproxy.yaml: -------------------------------------------------------------------------------- 1 | ############### epproxy configuration file ############### 2 | 3 | ####################### 4 | # APPLICATION OPTIONS # 5 | ####################### 6 | 7 | # proxy to run 8 | proxy: { 9 | # address and port to listen (0.0.0.0 for INADDR_ANY) 10 | listen: "0.0.0.0:80", 11 | 12 | # balancing algorithm to use 13 | # (any of "roundrobin", "least", "uri", or "single"); 14 | balance: roundrobin, 15 | 16 | # address and port of a backend server 17 | # (there should be a single backend server for "single" mode) 18 | backend: [ 19 | [name: "s1", addr: "10.0.0.1:80", weight: 1], 20 | [name: "s2", addr: "10.0.0.2:80", weight: 1], 21 | [name: "s3", addr: "10.0.0.3:80", weight: 1], 22 | [name: "s4", addr: "10.0.0.4:80", weight: 1], 23 | [name: "s5", addr: "10.0.0.5:80", weight: 1] 24 | ], 25 | 26 | # number of pre-established connections per backend server 27 | # (e.g., 128 conns/backend server * 2 backend servers = total 256 conns) 28 | conn_pool: 0, 29 | 30 | # method to maintain session persistence 31 | # (any of "none", "set-cookie", "append-cookie", "learn-cookie") 32 | # persist: set-cookie SERVERID, 33 | # persist: append-cookie JSESSIONID, 34 | # persist: learn-cookie ASP.NET_SessionId, 35 | } -------------------------------------------------------------------------------- /nic/scripts/nfp-sdk6-rte.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Netronome SDK6 Run Time Environment NORMAL MODE 3 | After=network.target 4 | 5 | [Service] 6 | Environment=CTL_SCRIPT=/opt/nfp_pif/scripts/pif_ctl_nfd.sh 7 | #To load RTE at startup set FW_FILE, DESIGN_FILE and CFG_FILE and set in ExecStart 8 | #Environment=FW_FILE= 9 | #Environment=DESIGN_FILE= 10 | #Environemnt=CFG_FILE= 11 | Environment=LOAD_NETDEV=1 12 | Environment=NUM_VFS=4 13 | #For custom DMA and MAC setting set NBI_DMA8_JSON and NBI_MAC8_JSON and set DETECT_MAC=no 14 | #Environment=NBI_DMA8_JSON=/opt/nfp_pif/etc/configs/platform_dma8_config.json 15 | #Environment=NBI_MAC8_JSON=/opt/nfp_pif/etc/configs/platform_mac8_config.json 16 | #Environment=NBI_TM_JSON=nfp_nbi_tm_12x10GE.json 17 | Environment=NFPSHUTILS=/opt/nfp_pif/scripts/shared/nfp-shutils 18 | Environment=DISABLE_NFD=no 19 | Environment=DETECT_MAC=yes 20 | Environment=LD_LIBRARY_PATH=/opt/netronome/lib:/opt/nfp_pif/lib:$LD_LIBRARY_PATH 21 | ExecStart=/opt/nfp_pif/bin/pif_rte -n 0 -p 20206 -I -s $CTL_SCRIPT --log_file /var/log/nfp-sdk6-rte.log 22 | #ExecStart=/opt/nfp_pif/bin/pif_rte -n 0 -p 20206 -I -z -s $CTL_SCRIPT -f $FW_FILE -d $DESIGN_FILE -c $CFG_FILE --log_file /var/log/nfp-sdk6-rte.log 23 | 24 | [Install] 25 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /apps/epproxy/src/http_parser.h: -------------------------------------------------------------------------------- 1 | #ifndef __HTTP_PARSER_H_ 2 | #define __HTTP_PARSER_H_ 3 | #include 4 | #include 5 | #include 6 | #include "http_stream.h" 7 | /*----------------------------------------------------------------------------*/ 8 | #define HTTP_DEFAULT_VER "1.1" /* epproxy uses HTTP/1.1 by default */ 9 | enum {HTTP_VER_1_0, HTTP_VER_1_1}; /* and we don't support HTTP/2 for now */ 10 | /*----------------------------------------------------------------------------*/ 11 | /* HTTP message parsing result */ 12 | enum {RR_MORE, 13 | RR_DONE, 14 | RR_ERROR_RESPONSE, /* wrong format or invalid value in server response */ 15 | RR_RETURN_400, 16 | RR_RETURN_501, 17 | RR_RETURN_505}; 18 | /*----------------------------------------------------------------------------*/ 19 | #define CRLFCRLF "\r\n\r\n" 20 | #define LFLF "\n\n" 21 | #define CRLF "\r\n" 22 | #define LF "\n" 23 | /*----------------------------------------------------------------------------*/ 24 | int 25 | ParseHTTPRequest(struct http_stream *hs, char **host_pos); 26 | /*----------------------------------------------------------------------------*/ 27 | int 28 | ParseHTTPResponse(struct http_stream *hs, struct http_stream *peer_hs); 29 | /*----------------------------------------------------------------------------*/ 30 | #endif 31 | -------------------------------------------------------------------------------- /mtcp/src/include/eth_out.h: -------------------------------------------------------------------------------- 1 | #ifndef __ETH_OUT_H_ 2 | #define __ETH_OUT_H_ 3 | 4 | #include 5 | 6 | #include "mtcp.h" 7 | #include "tcp_stream.h" 8 | #ifndef DISABLE_PSIO 9 | #include "ps.h" 10 | #endif 11 | 12 | #define MAX_SEND_PCK_CHUNK 64 13 | 14 | /*------------------------------------------------------------------------------------------*/ 15 | /* ethernet type used for NIC offload (setup, splice, teardown) */ 16 | #define ETH_P_IP_TCP_SETUP_OFFLOAD 0x0809 17 | #define ETH_P_IP_TCP_OFFLOAD_LISTEN 0x080B 18 | #define ETH_P_IP_TCP_SPLICE_OFFLOAD 0x0807 19 | #define CTRL_P_SPLICE_FINISH 0x0901 20 | #define DISABLE_HOST_ACK FALSE 21 | #define ETH_P_IP_TCP_TEARDOWN_OFFLOAD 0x0808 22 | #define ETH_P_IP_TCP_OFFLOAD_INIT_CTRL 0x080C 23 | /*------------------------------------------------------------------------------------------*/ 24 | 25 | uint8_t * 26 | EthernetOutput(struct mtcp_manager *mtcp, uint16_t h_proto, 27 | int nif, unsigned char* dst_haddr, uint16_t iplen, 28 | struct mtcp_offload_meta* offload_meta); 29 | 30 | #ifdef USE_NFP_NIC 31 | uint8_t * 32 | EthernetControlOutput(struct mtcp_manager *mtcp); 33 | uint8_t * 34 | AddListenNICPort(struct mtcp_manager *mtcp, uint32_t ip, uint16_t port); 35 | uint8_t * 36 | DelListenNICPort(struct mtcp_manager *mtcp); 37 | #endif 38 | 39 | #endif /* __ETH_OUT_H_ */ 40 | -------------------------------------------------------------------------------- /init_nic.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | if [ -f /lib/systemd/system/nfp-sdk6-rte.service ]; then 4 | cp /lib/systemd/system/nfp-sdk6-rte.service /lib/systemd/system/nfp-sdk6-rte.service.backup 5 | cp nic/scripts/nfp-sdk6-rte.service /lib/systemd/system/nfp-sdk6-rte.service 6 | else 7 | echo "Cannot find /lib/systemd/system/nfp-sdk6-rte.service.. please check if NFP SDK and BSP is installed" 8 | exit 1 9 | fi 10 | 11 | if [ -f /opt/nfp_pif/scripts/pif_ctl_nfd.sh ]; then 12 | cp /opt/nfp_pif/scripts/pif_ctl_nfd.sh /opt/nfp_pif/scripts/pif_ctl_nfd.sh.backup 13 | cp nic/scripts/pif_ctl_nfd.sh /opt/nfp_pif/scripts/pif_ctl_nfd.sh 14 | cp nic/scripts/dpdk-devbind.py /opt/nfp_pif/scripts/dpdk-devbind.py 15 | else 16 | echo "Cannot find /opt/nfp_pif/scripts/pif_ctl_nfd.sh.. please check if NFP SDK and BSP is installed" 17 | exit 1 18 | fi 19 | 20 | cd nic/setup_teardown; ./build_p4; cd ../.. 21 | if [ ! -f nic/setup_teardown/src/build/dataplane.nffw ]; then 22 | echo "Failed to build NIC dataplane for connection setup and teardown offload" 23 | exit 1 24 | fi 25 | 26 | cd nic/splice; ./build_p4; cd ../.. 27 | if [ ! -f nic/splice/src/build/splice.nffw ]; then 28 | echo "Failed to build NIC dataplane for connection splicing offload" 29 | exit 1 30 | fi 31 | 32 | echo "---------------------------------------" 33 | echo "Finished initializing runtime services/scripts and building NIC dataplane" 34 | echo "---------------------------------------" 35 | -------------------------------------------------------------------------------- /nic/splice/config/default.p4cfg: -------------------------------------------------------------------------------- 1 | { 2 | "tables": { 3 | "fwd_tbl": { 4 | "rules": [ 5 | { 6 | "action": { 7 | "type": "fwd_to_host" 8 | }, 9 | "name": "from_nic_0", 10 | "match": { 11 | "standard_metadata.ingress_port": { 12 | "value": "p0" 13 | } 14 | } 15 | }, 16 | { 17 | "action": { 18 | "type": "fwd_to_host" 19 | }, 20 | "name": "from_nic_1", 21 | "match": { 22 | "standard_metadata.ingress_port": { 23 | "value": "p4" 24 | } 25 | } 26 | }, 27 | { 28 | "action": { 29 | "type": "strip_and_fwd_to_network" 30 | }, 31 | "name": "from_host_offload", 32 | "match": { 33 | "offload": { 34 | "value": "valid" 35 | } 36 | } 37 | } 38 | ], 39 | "default_rule": { 40 | "action": { 41 | "type": "fwd_to_network" 42 | }, 43 | "name": "from_host" 44 | } 45 | } 46 | } 47 | } -------------------------------------------------------------------------------- /mtcp/src/include/timer.h: -------------------------------------------------------------------------------- 1 | #ifndef __TIMER_H_ 2 | #define __TIMER_H_ 3 | 4 | #include "mtcp.h" 5 | #include "tcp_stream.h" 6 | 7 | #define RTO_HASH 3000 8 | 9 | struct rto_hashstore 10 | { 11 | uint32_t rto_now_idx; // pointing the hs_table_s index 12 | uint32_t rto_now_ts; // 13 | 14 | TAILQ_HEAD(rto_head , tcp_stream) rto_list[RTO_HASH+1]; 15 | }; 16 | 17 | struct rto_hashstore* 18 | InitRTOHashstore(); 19 | 20 | extern inline void 21 | AddtoRTOList(mtcp_manager_t mtcp, tcp_stream *cur_stream); 22 | 23 | extern inline void 24 | RemoveFromRTOList(mtcp_manager_t mtcp, tcp_stream *cur_stream); 25 | 26 | extern inline void 27 | AddtoTimewaitList(mtcp_manager_t mtcp, tcp_stream *cur_stream, uint32_t cur_ts); 28 | 29 | extern inline void 30 | RemoveFromTimewaitList(mtcp_manager_t mtcp, tcp_stream *cur_stream); 31 | 32 | extern inline void 33 | AddtoTimeoutList(mtcp_manager_t mtcp, tcp_stream *cur_stream); 34 | 35 | extern inline void 36 | RemoveFromTimeoutList(mtcp_manager_t mtcp, tcp_stream *cur_stream); 37 | 38 | extern inline void 39 | UpdateTimeoutList(mtcp_manager_t mtcp, tcp_stream *cur_stream); 40 | 41 | extern inline void 42 | UpdateRetransmissionTimer(mtcp_manager_t mtcp, 43 | tcp_stream *cur_stream, uint32_t cur_ts); 44 | 45 | void 46 | CheckRtmTimeout(mtcp_manager_t mtcp, uint32_t cur_ts, int thresh); 47 | 48 | void 49 | CheckTimewaitExpire(mtcp_manager_t mtcp, uint32_t cur_ts, int thresh); 50 | 51 | void 52 | CheckConnectionTimeout(mtcp_manager_t mtcp, uint32_t cur_ts, int thresh); 53 | 54 | #endif /* __TIMER_H_ */ 55 | -------------------------------------------------------------------------------- /apps/epproxy/src/persist.h: -------------------------------------------------------------------------------- 1 | #ifndef HTTP_PERSIST_H 2 | #define HTTP_PERSIST_H 3 | #include 4 | #include "http_stream.h" 5 | #include "backend_pool.h" 6 | #include "config.h" 7 | /*----------------------------------------------------------------------------*/ 8 | enum {RES_RUN_LB, 9 | RES_FOUND_PERSIST, 10 | RES_ADD_STICKY_TABLE}; 11 | /*----------------------------------------------------------------------------*/ 12 | enum {PM_NONE, 13 | PM_SET_COOKIE, 14 | PM_APPEND_COOKIE, 15 | PM_LEARN_COOKIE}; 16 | /*----------------------------------------------------------------------------*/ 17 | #define COOKIE_VALUE_LEN 100 18 | /*----------------------------------------------------------------------------*/ 19 | typedef struct sticky_ent { 20 | char value[COOKIE_VALUE_LEN + 1]; 21 | struct backend_info* backend; 22 | 23 | TAILQ_ENTRY (sticky_ent) link; 24 | 25 | } sticky_ent; 26 | /*----------------------------------------------------------------------------*/ 27 | TAILQ_HEAD (sticky_table, sticky_ent); /* sticky table */ 28 | /*----------------------------------------------------------------------------*/ 29 | int 30 | HandleRequestPersistence(struct sticky_table *sticky_map, 31 | struct http_stream *hs); 32 | /*----------------------------------------------------------------------------*/ 33 | int 34 | HandleResponsePersistence(struct sticky_table *sticky_map, 35 | struct http_stream *hs, 36 | struct http_stream *peer_hs, 37 | char *sticky_value); 38 | /*----------------------------------------------------------------------------*/ 39 | #endif 40 | -------------------------------------------------------------------------------- /apps/epproxy/src/http_stream.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include "backend_pool.h" 3 | #ifndef HTTP_STREAM_H 4 | #define HTTP_STREAM_H 5 | /*----------------------------------------------------------------------------*/ 6 | typedef struct http_buf { 7 | int cnt_refs; /* number of references by http_stream */ 8 | char *data; /* payload buffer */ 9 | int data_len; /* bytes used in the payload buffer */ 10 | 11 | TAILQ_ENTRY (http_buf) link; 12 | 13 | } http_buf; 14 | /*----------------------------------------------------------------------------*/ 15 | typedef struct http_stream { 16 | int sock; /* socket of itself */ 17 | int peer_sock; /* socket to its peer (frontend <-> backend) */ 18 | int write_blocked; /* whether its socket is blocked to write */ 19 | int keep_alive; /* (frontend) whether it is a keep-alive connection */ 20 | int wait_header; /* true if waiting for header */ 21 | char* uri; 22 | 23 | int is_front; 24 | int64_t bytes_to_write; /* (frontend) bytes to write */ 25 | 26 | http_buf *rbuf; /* read buffer for its http payload */ 27 | http_buf *wbuf; /* write buffer for its http payload */ 28 | 29 | struct backend_info* backend; 30 | 31 | int is_spare; /* connections that can be reused */ 32 | int is_health_check; /* connections used for health check */ 33 | int is_connecting; /* try connecting to backend server */ 34 | 35 | int nif_out; 36 | 37 | 38 | TAILQ_ENTRY (http_stream) link; 39 | 40 | } http_stream; 41 | /*----------------------------------------------------------------------------*/ 42 | #endif 43 | -------------------------------------------------------------------------------- /nic/setup_teardown/src/include/nfp_device.h: -------------------------------------------------------------------------------- 1 | /* Agilio NFP NIC-specific variables */ 2 | #ifndef NFP_DEVICE_H 3 | #define NFP_DEVICE_H 4 | /*-------------------------------------------------------------------------*/ 5 | #define MASTER_MEID 0x0204 6 | #define MASTER_CTX 0 7 | /*-------------------------------------------------------------------------*/ 8 | #define BASE_ISLAND_ID 32 9 | #define BASE_MEID_OFFSET 4 10 | #define MAX_ME_PER_ISLAND 12 11 | #define MAX_ISLANDS 7 12 | /*-------------------------------------------------------------------------*/ 13 | /* calculate continuous meid starting from zero */ 14 | void 15 | nfp_dev_set_cont_meid() { 16 | 17 | /* ygmoon: convert device specific IDs to continuous meid 18 | * (i32.me0: 0, i32.me1: 1, .., i33.me11: MAX_ME_PER_ISLAND = 12, ..) */ 19 | uint32_t cont_meid = (__ISLAND - BASE_ISLAND_ID) * MAX_ME_PER_ISLAND 20 | + ((__MEID & 0xF) - BASE_MEID_OFFSET); 21 | 22 | /* ygmoon: adjust cont_meid to be continuous 23 | * (note that i33.me10, i33.me11, i36.me11 are not used in nfp6000) */ 24 | if (cont_meid > 0x15) 25 | cont_meid -= 2; 26 | if (cont_meid > 0x38) 27 | cont_meid -= 1; 28 | 29 | /* store continuous meid at local_csr_mailbox_2 */ 30 | local_csr_write(local_csr_mailbox_2, cont_meid); 31 | } 32 | /*-------------------------------------------------------------------------*/ 33 | uint32_t 34 | nfp_dev_get_cont_meid() { 35 | 36 | /* load continuous meid from local_csr_mailbox_2 */ 37 | return local_csr_read(local_csr_mailbox_2); 38 | } 39 | /*-------------------------------------------------------------------------*/ 40 | #endif 41 | -------------------------------------------------------------------------------- /mtcp/src/include/eventpoll.h: -------------------------------------------------------------------------------- 1 | #ifndef __EVENTPOLL_H_ 2 | #define __EVENTPOLL_H_ 3 | 4 | #include "mtcp_api.h" 5 | #include "mtcp_epoll.h" 6 | 7 | /*----------------------------------------------------------------------------*/ 8 | struct mtcp_epoll_stat 9 | { 10 | uint64_t calls; 11 | uint64_t waits; 12 | uint64_t wakes; 13 | 14 | uint64_t issued; 15 | uint64_t registered; 16 | uint64_t invalidated; 17 | uint64_t handled; 18 | }; 19 | /*----------------------------------------------------------------------------*/ 20 | struct mtcp_epoll_event_int 21 | { 22 | struct mtcp_epoll_event ev; 23 | int sockid; 24 | }; 25 | /*----------------------------------------------------------------------------*/ 26 | enum event_queue_type 27 | { 28 | USR_EVENT_QUEUE = 0, 29 | USR_SHADOW_EVENT_QUEUE = 1, 30 | MTCP_EVENT_QUEUE = 2 31 | }; 32 | /*----------------------------------------------------------------------------*/ 33 | struct event_queue 34 | { 35 | struct mtcp_epoll_event_int *events; 36 | int start; // starting index 37 | int end; // ending index 38 | 39 | int size; // max size 40 | int num_events; // number of events 41 | }; 42 | /*----------------------------------------------------------------------------*/ 43 | struct mtcp_epoll 44 | { 45 | struct event_queue *usr_queue; 46 | struct event_queue *usr_shadow_queue; 47 | struct event_queue *mtcp_queue; 48 | 49 | uint8_t waiting; 50 | struct mtcp_epoll_stat stat; 51 | 52 | pthread_cond_t epoll_cond; 53 | pthread_mutex_t epoll_lock; 54 | }; 55 | /*----------------------------------------------------------------------------*/ 56 | 57 | int 58 | CloseEpollSocket(mctx_t mctx, int epid); 59 | 60 | #endif /* __EVENTPOLL_H_ */ 61 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | AccelTCP source code is distributed under the Modified BSD Licence. 2 | 3 | Copyright (C) 2020 YoungGyoun Moon, SeungEon Lee, 4 | Muhammad Asim Jamshed, KyoungSoo Park 5 | 6 | All rights reserved. 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are met: 10 | * Redistributions of source code must retain the above copyright 11 | notice, this list of conditions and the following disclaimer. 12 | * Redistributions in binary form must reproduce the above copyright 13 | notice, this list of conditions and the following disclaimer in the 14 | documentation and/or other materials provided with the distribution. 15 | * Neither the name of the nor the 16 | names of its contributors may be used to endorse or promote products 17 | derived from this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY 23 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 26 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /mtcp/src/include/fhash.h: -------------------------------------------------------------------------------- 1 | #ifndef __FHASH_H_ 2 | #define __FHASH_H_ 3 | 4 | #include 5 | #include "tcp_stream.h" 6 | 7 | #define NUM_BINS_FLOWS (131072) /* 132 K entries per thread*/ 8 | #define NUM_BINS_LISTENERS (1024) /* assuming that chaining won't happen excessively */ 9 | #define TCP_AR_CNT (3) 10 | 11 | typedef struct hash_bucket_head { 12 | tcp_stream *tqh_first; 13 | tcp_stream **tqh_last; 14 | } hash_bucket_head; 15 | 16 | typedef struct list_bucket_head { 17 | struct tcp_listener *tqh_first; 18 | struct tcp_listener **tqh_last; 19 | } list_bucket_head; 20 | 21 | /* hashtable structure */ 22 | struct hashtable { 23 | uint8_t ht_count ; // count for # entry 24 | uint32_t bins; 25 | 26 | union { 27 | hash_bucket_head *ht_table; 28 | list_bucket_head *lt_table; 29 | }; 30 | 31 | // functions 32 | unsigned int (*hashfn) (const void *); 33 | int (*eqfn) (const void *, const void *); 34 | }; 35 | 36 | /*functions for hashtable*/ 37 | struct hashtable *CreateHashtable(unsigned int (*hashfn) (const void *), 38 | int (*eqfn) (const void *, 39 | const void *), 40 | int bins); 41 | void DestroyHashtable(struct hashtable *ht); 42 | 43 | 44 | int StreamHTInsert(struct hashtable *ht, void *); 45 | void* StreamHTRemove(struct hashtable *ht, void *); 46 | void *StreamHTSearch(struct hashtable *ht, const void *); 47 | unsigned int HashListener(const void *hbo_port_ptr); 48 | int EqualListener(const void *hbo_port_ptr1, const void *hbo_port_ptr2); 49 | int ListenerHTInsert(struct hashtable *ht, void *); 50 | void *ListenerHTRemove(struct hashtable *ht, void *); 51 | void *ListenerHTSearch(struct hashtable *ht, const void *); 52 | 53 | #endif /* __FHASH_H_ */ 54 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | AUTOMAKE_OPTIONS = foreign 2 | SUBDIRS = mtcp/src apps/example apps/epproxy #io_engine/lib 3 | 4 | install-man: 5 | @echo -e "\e[1;34mPlacing the man pages in the $(prefix)/man/man3/ directory $<\e[0m" 6 | @ mkdir -p $(prefix)/man/ 7 | @ mkdir -p $(prefix)/man/man3/ 8 | @ cp docs/man/* $(prefix)/man/man3/ 9 | @echo -e "\e[1;34mUpdating man pages cache $<\e[0m" 10 | @ mandb > /dev/null 2> /dev/null 11 | 12 | uninstall: 13 | @echo -e "\e[1;34mCleaning up... $<\e[0m" 14 | @ rm -rf $(prefix)/mtcp 15 | @ rm -rf $(prefix)/man/man3/mtcp_* 16 | @echo -e "\e[1;34mUpdating man pages cache $<\e[0m" 17 | @ mandb > /dev/null 2> /dev/null 18 | 19 | if DPDK 20 | install: all install-man 21 | @echo -e "\e[1;34mPlacing the libraries in the $(prefix)/mtcp/lib directory $<\e[0m" 22 | @ mkdir -p $(prefix)/mtcp/ 23 | @ mkdir -p $(prefix)/mtcp/lib 24 | @ mkdir -p $(prefix)/mtcp/bin 25 | @ mkdir -p $(prefix)/mtcp/include 26 | @ cp mtcp/lib/libmtcp.a $(prefix)/mtcp/lib/ 27 | @ cp dpdk/lib/* $(prefix)/mtcp/lib/ 28 | @ cp -rL dpdk/include/* $(prefix)/mtcp/include/ 29 | @ cp mtcp/include/* $(prefix)/mtcp/include/ 30 | @echo -e "\e[1;34mPlacing the binaries in the $(prefix)/mtcp/bin directory $<\e[0m" 31 | @ cp `find apps/example/ -type f -executable -perm /a+x` $(prefix)/mtcp/bin/ 32 | else 33 | install: all install-man 34 | @echo -e "\e[1;34mPlacing the libraries in the $(prefix)/mtcp/lib directory $<\e[0m" 35 | @ mkdir -p $(prefix)/mtcp/ 36 | @ mkdir -p $(prefix)/mtcp/lib 37 | @ mkdir -p $(prefix)/mtcp/bin 38 | @ mkdir -p $(prefix)/mtcp/include 39 | @ cp mtcp/lib/libmtcp.a $(prefix)/mtcp/lib/ 40 | @ cp mtcp/include/* $(prefix)/mtcp/include/ 41 | @echo -e "\e[1;34mPlacing the binaries in the $(prefix)/mtcp/bin directory $<\e[0m" 42 | @ cp `find apps/example/ -type f -executable -perm /a+x` $(prefix)/mtcp/bin/ 43 | endif 44 | -------------------------------------------------------------------------------- /mtcp/src/eth_in.c: -------------------------------------------------------------------------------- 1 | #ifndef DISABLE_PSIO 2 | #include "ps.h" 3 | #endif 4 | #include "ip_in.h" 5 | #include "eth_in.h" 6 | #include "eth_out.h" 7 | #include "arp.h" 8 | #include "nic_control.h" 9 | #include "debug.h" 10 | 11 | /*----------------------------------------------------------------------------*/ 12 | int 13 | ProcessPacket(mtcp_manager_t mtcp, const int ifidx, 14 | uint32_t cur_ts, unsigned char *pkt_data, int len) 15 | { 16 | struct ethhdr *ethh = (struct ethhdr *)pkt_data; 17 | u_short ip_proto = ntohs(ethh->h_proto); 18 | int ret; 19 | 20 | #ifdef PKTDUMP 21 | DumpPacket(mtcp, (char *)pkt_data, len, "IN", ifidx); 22 | #endif 23 | 24 | #ifdef NETSTAT 25 | mtcp->nstat.rx_packets[ifidx]++; 26 | mtcp->nstat.rx_bytes[ifidx] += len + 24; 27 | #endif /* NETSTAT */ 28 | 29 | #if 0 30 | /* ignore mac address which is not for current interface */ 31 | int i; 32 | for (i = 0; i < 6; i ++) { 33 | if (ethh->h_dest[i] != CONFIG.eths[ifidx].haddr[i]) { 34 | return FALSE; 35 | } 36 | } 37 | #endif 38 | 39 | if (ip_proto == ETH_P_IP_TCP_SETUP_OFFLOAD) { 40 | ret = ProcessSetupOffloadPacket(mtcp, cur_ts, ifidx, pkt_data, len); 41 | } 42 | else if (ip_proto == ETH_P_IP) { 43 | /* process ipv4 packet */ 44 | ret = ProcessIPv4Packet(mtcp, cur_ts, ifidx, pkt_data, len); 45 | } else if (ip_proto == CTRL_P_SPLICE_FINISH) { 46 | ret = ProcessSpliceFinishPacket(mtcp, cur_ts, ifidx, pkt_data, len); 47 | } else if (ip_proto == ETH_P_ARP) { 48 | ProcessARPPacket(mtcp, cur_ts, ifidx, pkt_data, len); 49 | return TRUE; 50 | } else { 51 | //DumpPacket(mtcp, (char *)pkt_data, len, "??", ifidx); 52 | mtcp->iom->release_pkt(mtcp->ctx, ifidx, pkt_data, len); 53 | return TRUE; 54 | } 55 | 56 | #ifdef NETSTAT 57 | if (ret < 0) { 58 | mtcp->nstat.rx_errors[ifidx]++; 59 | } 60 | #endif 61 | 62 | return ret; 63 | } 64 | -------------------------------------------------------------------------------- /mtcp/src/include/tcp_send_buffer.h: -------------------------------------------------------------------------------- 1 | #ifndef __TCP_SEND_BUFFER_H_ 2 | #define __TCP_SEND_BUFFER_H_ 3 | 4 | #include 5 | #include 6 | 7 | #define NEW_SB 0 /* a new tcp_send_buffer implementation w/o memmove() */ 8 | 9 | /*----------------------------------------------------------------------------*/ 10 | typedef struct sb_manager* sb_manager_t; 11 | /*----------------------------------------------------------------------------*/ 12 | struct tcp_send_buffer 13 | { 14 | unsigned char *data; 15 | unsigned char *head; 16 | 17 | uint32_t head_off; 18 | uint32_t tail_off; 19 | uint32_t len; 20 | #ifdef ENABLE_LOGGER 21 | uint64_t cum_len; 22 | #endif 23 | uint32_t size; 24 | 25 | uint32_t head_seq; 26 | uint32_t init_seq; 27 | }; 28 | /*----------------------------------------------------------------------------*/ 29 | uint32_t 30 | SBGetCurnum(sb_manager_t sbm); 31 | /*----------------------------------------------------------------------------*/ 32 | sb_manager_t 33 | SBManagerCreate(size_t chunk_size, uint32_t cnum); 34 | /*----------------------------------------------------------------------------*/ 35 | struct tcp_send_buffer * 36 | SBInit(sb_manager_t sbm, uint32_t init_seq); 37 | /*----------------------------------------------------------------------------*/ 38 | void 39 | SBFree(sb_manager_t sbm, struct tcp_send_buffer *buf); 40 | /*----------------------------------------------------------------------------*/ 41 | size_t 42 | SBPut(sb_manager_t sbm, struct tcp_send_buffer *buf, const void *data, size_t len); 43 | /*----------------------------------------------------------------------------*/ 44 | size_t 45 | SBRemove(sb_manager_t sbm, struct tcp_send_buffer *buf, size_t len); 46 | /*----------------------------------------------------------------------------*/ 47 | 48 | #endif /* __TCP_SEND_BUFFER_H_ */ 49 | -------------------------------------------------------------------------------- /mtcp/src/include/stat.h: -------------------------------------------------------------------------------- 1 | #ifndef __STAT_H_ 2 | #define __STAT_H_ 3 | 4 | #include "config.h" 5 | 6 | struct run_stat 7 | { 8 | uint64_t rounds; 9 | uint64_t rounds_rx; 10 | uint64_t rounds_rx_try; 11 | uint64_t rounds_tx; 12 | uint64_t rounds_tx_try; 13 | uint64_t rounds_select; 14 | uint64_t rounds_select_rx; 15 | uint64_t rounds_select_tx; 16 | uint64_t rounds_select_intr; 17 | 18 | uint64_t rounds_accept; 19 | uint64_t rounds_read; 20 | uint64_t rounds_write; 21 | uint64_t rounds_epoll; 22 | uint64_t rounds_wndadv; 23 | 24 | uint64_t rounds_rtocheck; 25 | uint64_t rounds_twcheck; 26 | uint64_t rounds_tocheck; 27 | }; 28 | 29 | struct stat_counter 30 | { 31 | uint64_t cnt; 32 | uint64_t sum; 33 | uint64_t max; 34 | uint64_t min; 35 | }; 36 | 37 | struct time_stat 38 | { 39 | struct stat_counter round; 40 | struct stat_counter processing; 41 | struct stat_counter tcheck; 42 | struct stat_counter epoll; 43 | struct stat_counter handle; 44 | struct stat_counter xmit; 45 | struct stat_counter select; 46 | }; 47 | 48 | struct net_stat 49 | { 50 | uint64_t tx_packets[MAX_DEVICES]; 51 | uint64_t tx_bytes[MAX_DEVICES]; 52 | uint64_t tx_drops[MAX_DEVICES]; 53 | uint64_t rx_packets[MAX_DEVICES]; 54 | uint64_t rx_bytes[MAX_DEVICES]; 55 | uint64_t rx_errors[MAX_DEVICES]; 56 | #ifdef ENABLELRO 57 | uint64_t tx_gdptbytes; 58 | uint64_t rx_gdptbytes; 59 | #endif 60 | }; 61 | 62 | struct bcast_stat 63 | { 64 | uint64_t cycles; 65 | uint64_t write; 66 | uint64_t read; 67 | uint64_t epoll; 68 | uint64_t wnd_adv; 69 | uint64_t ack; 70 | }; 71 | 72 | struct timeout_stat 73 | { 74 | uint64_t cycles; 75 | uint64_t rto_try; 76 | uint64_t rto; 77 | uint64_t timewait_try; 78 | uint64_t timewait; 79 | }; 80 | 81 | #ifdef NETSTAT 82 | #define STAT_COUNT(stat) stat++ 83 | #else 84 | #define STAT_COUNT(stat) 85 | #endif 86 | 87 | #endif /* __STAT_H_ */ 88 | -------------------------------------------------------------------------------- /apps/epproxy/config/sample_mtcp.conf: -------------------------------------------------------------------------------- 1 | ############### mtcp configuration file ############### 2 | 3 | # The underlying I/O module you want to use. Please 4 | # enable only one out of the two. 5 | #io = psio 6 | #io = netmap 7 | io = dpdk 8 | 9 | # No. of cores setting (enabling this option will override 10 | # the `cpu' config for those applications that accept 11 | # num_cores as command line arguments) 12 | # 13 | # e.g. in case ./epserver is executed with `-N 4', the 14 | # mtcp core will still invoke 8 mTCP threads if the 15 | # following line is uncommented. 16 | #num_cores = 8 17 | 18 | # Enable multi-process support (currently under development) 19 | #multiprocess = 0 master 20 | #multiprocess = 1 21 | 22 | # Number of memory channels per processor socket (dpdk-only!) 23 | num_mem_ch = 4 24 | 25 | # Used port (please adjust accordingly) 26 | #------ PSIO ports -------# 27 | #port = xge0 xge1 28 | #port = xge1 29 | #------ DPDK ports -------# 30 | port = dpdk0 dpdk1 31 | #port = dpdk0 32 | #port = dpdk1 33 | 34 | # Maximum concurrency per core 35 | max_concurrency = 10000 36 | 37 | # Maximum number of socket buffers per core 38 | # Set this to small value if there are many idle connections 39 | max_num_buffers = 10000 40 | 41 | # Receive buffer size of sockets 42 | rcvbuf = 8192 43 | #rcvbuf = 16384 44 | 45 | # Send buffer size of sockets 46 | sndbuf = 8192 47 | #sndbuf = 2048 48 | 49 | # TCP timeout seconds 50 | # (tcp_timeout = -1 can disable the timeout check) 51 | tcp_timeout = 30 52 | 53 | # TCP timewait seconds 54 | tcp_timewait = 0 55 | 56 | # Interface to print stats (please adjust accordingly) 57 | # You can enable multiple ports in separate lines 58 | #------ PSIO ports -------# 59 | #stat_print = xge0 60 | #stat_print = xge1 61 | #------ DPDK ports -------# 62 | stat_print = dpdk0 63 | stat_print = dpdk1 64 | 65 | ####################################################### -------------------------------------------------------------------------------- /apps/example/config/sample_mtcp.conf: -------------------------------------------------------------------------------- 1 | ############### mtcp configuration file ############### 2 | 3 | # The underlying I/O module you want to use. Please 4 | # enable only one out of the two. 5 | #io = psio 6 | #io = netmap 7 | io = dpdk 8 | 9 | # No. of cores setting (enabling this option will override 10 | # the `cpu' config for those applications that accept 11 | # num_cores as command line arguments) 12 | # 13 | # e.g. in case ./epserver is executed with `-N 4', the 14 | # mtcp core will still invoke 8 mTCP threads if the 15 | # following line is uncommented. 16 | #num_cores = 8 17 | 18 | # Enable multi-process support (currently under development) 19 | #multiprocess = 0 master 20 | #multiprocess = 1 21 | 22 | # Number of memory channels per processor socket (dpdk-only!) 23 | num_mem_ch = 4 24 | 25 | # Used port (please adjust accordingly) 26 | #------ PSIO ports -------# 27 | #port = xge0 xge1 28 | #port = xge1 29 | #------ DPDK ports -------# 30 | port = dpdk0 dpdk1 31 | #port = dpdk0 32 | #port = dpdk1 33 | 34 | # Maximum concurrency per core 35 | max_concurrency = 10000 36 | 37 | # Maximum number of socket buffers per core 38 | # Set this to small value if there are many idle connections 39 | max_num_buffers = 10000 40 | 41 | # Receive buffer size of sockets 42 | rcvbuf = 8192 43 | #rcvbuf = 16384 44 | 45 | # Send buffer size of sockets 46 | sndbuf = 8192 47 | #sndbuf = 2048 48 | 49 | # TCP timeout seconds 50 | # (tcp_timeout = -1 can disable the timeout check) 51 | tcp_timeout = 30 52 | 53 | # TCP timewait seconds 54 | tcp_timewait = 0 55 | 56 | # Interface to print stats (please adjust accordingly) 57 | # You can enable multiple ports in separate lines 58 | #------ PSIO ports -------# 59 | #stat_print = xge0 60 | #stat_print = xge1 61 | #------ DPDK ports -------# 62 | stat_print = dpdk0 63 | stat_print = dpdk1 64 | 65 | ####################################################### -------------------------------------------------------------------------------- /apps/epproxy/src/balance.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "balance.h" 6 | #include "http_stream.h" 7 | #include "http_parser.h" 8 | #include "util.h" 9 | #include "hash.h" 10 | #include "persist.h" 11 | #include "epproxy.h" 12 | /*----------------------------------------------------------------------------*/ 13 | inline backend_info* 14 | FindBackendByURI(const char* uri, uint32_t uri_len) 15 | { 16 | uint32_t hash; 17 | 18 | /* calculate two-step hash functions */ 19 | hash = sdbm_hash(uri, uri_len); 20 | hash = full_aval_hash(hash); 21 | 22 | /* find for a corresponding server node */ 23 | return SelectNodeByHash(hash); 24 | } 25 | /*----------------------------------------------------------------------------*/ 26 | void 27 | DecideBackendServer(struct sticky_table *sticky_map, 28 | int* rr_count, struct http_stream *hs) 29 | { 30 | struct backend_pool* bpool = &g_prx_ctx->bpool[hs->nif_out]; 31 | size_t bpool_num = get_backend_poolsize(bpool); 32 | 33 | /* load balancing algorithm */ 34 | if (hs->backend == NULL) { 35 | if (g_prx_ctx->balance == BL_ROUNDROBIN) { 36 | hs->backend = get_server_from_bpool_by_pos(bpool, 37 | (*rr_count) % bpool_num); 38 | (*rr_count)++; 39 | } 40 | else if (g_prx_ctx->balance == BL_SINGLE) { 41 | hs->backend = get_server_from_bpool_by_pos(bpool, 0); 42 | } 43 | else if (g_prx_ctx->balance == BL_URI) { 44 | if (!hs->uri) { 45 | /* For now, we exit on this case for debugging.. */ 46 | exit(-1); 47 | } 48 | 49 | /* find for a backend server by URI */ 50 | hs->backend = FindBackendByURI(hs->uri, strlen(hs->uri)); 51 | } 52 | else { 53 | TRACE_ERROR("we don't support others for a while\n"); 54 | exit(-1); 55 | } 56 | } 57 | } 58 | /*----------------------------------------------------------------------------*/ 59 | -------------------------------------------------------------------------------- /apps/epproxy/Makefile.in: -------------------------------------------------------------------------------- 1 | TARGET = src/epproxy 2 | 3 | ###################################################################### 4 | # GCC and compilation options 5 | ###################################################################### 6 | GCC = $(CC) 7 | GCC_OPT = -m64 -Wall -Werror -fgnu89-inline 8 | GCC_OPT += -DNDEBUG -O3 -g -DNETSTAT -DINFO -DDBGERR -DDBGCERR 9 | #GCC_OPT += $(DBG_OPT) 10 | GCC_OPT += -DENABLE_UCTX 11 | ifeq ($V,) # no echo 12 | export MSG=@echo 13 | export HIDE=@ 14 | else 15 | export MSG=@\# 16 | export HIDE= 17 | endif 18 | 19 | ###################################################################### 20 | # LIBRARIES AND INCLUDES 21 | ###################################################################### 22 | MTCP_FLD = ../../mtcp 23 | MTCP_INC =-I$(MTCP_FLD)/include 24 | MTCP_TARGET = $(MTCP_FLD)/lib/libmtcp.a 25 | YAML_FLD = lib/yaml-0.1.7 26 | YAML_INC =-I$(YAML_FLD)/include 27 | YAML_TARGET = $(YAML_FLD)/libyaml.a 28 | LIBS += -lmtcp -lnuma -lpthread -lrt # -lssl -lcrypto 29 | LIB_DIR += -L$(MTCP_FLD)/lib 30 | #CMN_DIR = ../common 31 | #CMN_INC = -I$(CMN_DIR) 32 | #UTIL_INC = -I../../util/include 33 | 34 | DPDK_LIB=@DPDKLIBPATH@/lib/ 35 | DPDK_LIB_FLAGS = $(shell cat @DPDKLIBPATH@/lib/ldflags.txt) 36 | LIBS += -m64 -g -pthread -lrt -march=native -Wl,-export-dynamic \ 37 | -Wl,-lnuma \ 38 | -Wl,-lmtcp -Wl,-lpthread -Wl,-lrt -Wl,-ldl ${DPDK_LIB_FLAGS} 39 | ###################################################################### 40 | #-L../..//dpdk/lib 41 | 42 | default: $(TARGET) 43 | 44 | $(MTCP_TARGET): 45 | cd $(MTCP_FLD)/src && make 46 | 47 | $(TARGET): $(MTCP_TARGET) $(TARGET).c src/*.c $(YAML_TARGET) #$(CMN_DIR)/*.c 48 | $(MSG) " CC $<" 49 | $(HIDE) $(GCC) $(GCC_OPT) -o $@ $^ $(MTCP_INC) $(YAML_INC) $(LIB_DIR) $(LIBS) 50 | mv $@ . 51 | 52 | clean: 53 | rm -rf *~ *.o ./epproxy logs/* 54 | cd $(MTCP_FLD)/src && make clean 55 | 56 | -------------------------------------------------------------------------------- /mtcp/src/include/socket.h: -------------------------------------------------------------------------------- 1 | #ifndef __SOCKET_H_ 2 | #define __SOCKET_H_ 3 | 4 | #include "mtcp_api.h" 5 | #include "mtcp_epoll.h" 6 | 7 | /*----------------------------------------------------------------------------*/ 8 | enum socket_opts 9 | { 10 | MTCP_NONBLOCK = 0x01, 11 | MTCP_ADDR_BIND = 0x02, 12 | }; 13 | /*----------------------------------------------------------------------------*/ 14 | struct socket_map 15 | { 16 | int id; 17 | int socktype; 18 | uint32_t opts; 19 | 20 | struct sockaddr_in saddr; 21 | 22 | union { 23 | struct tcp_stream *stream; 24 | struct tcp_listener *listener; 25 | struct mtcp_epoll *ep; 26 | struct pipe *pp; 27 | }; 28 | 29 | uint32_t epoll; /* registered events */ 30 | uint32_t events; /* available events */ 31 | mtcp_epoll_data_t ep_data; 32 | 33 | TAILQ_ENTRY (socket_map) free_smap_link; 34 | 35 | }; 36 | /*----------------------------------------------------------------------------*/ 37 | typedef struct socket_map * socket_map_t; 38 | /*----------------------------------------------------------------------------*/ 39 | socket_map_t 40 | AllocateSocket(mctx_t mctx, int socktype, int need_lock); 41 | /*----------------------------------------------------------------------------*/ 42 | void 43 | FreeSocket(mctx_t mctx, int sockid, int need_lock); 44 | /*----------------------------------------------------------------------------*/ 45 | socket_map_t 46 | GetSocket(mctx_t mctx, int sockid); 47 | /*----------------------------------------------------------------------------*/ 48 | struct tcp_listener 49 | { 50 | int sockid; 51 | socket_map_t socket; 52 | 53 | int backlog; 54 | stream_queue_t acceptq; 55 | 56 | pthread_mutex_t accept_lock; 57 | pthread_cond_t accept_cond; 58 | 59 | TAILQ_ENTRY(tcp_listener) he_link; /* hash table entry link */ 60 | }; 61 | /*----------------------------------------------------------------------------*/ 62 | 63 | #endif /* __SOCKET_H_ */ 64 | -------------------------------------------------------------------------------- /mtcp/src/cpu.c: -------------------------------------------------------------------------------- 1 | #ifndef _GNU_SOURCE 2 | #define _GNU_SOURCE 3 | #endif 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "mtcp_api.h" 14 | 15 | #define MAX_FILE_NAME 1024 16 | 17 | /*----------------------------------------------------------------------------*/ 18 | int 19 | GetNumCPUs() 20 | { 21 | return sysconf(_SC_NPROCESSORS_ONLN); 22 | } 23 | /*----------------------------------------------------------------------------*/ 24 | pid_t 25 | Gettid() 26 | { 27 | return syscall(__NR_gettid); 28 | } 29 | /*----------------------------------------------------------------------------*/ 30 | int 31 | mtcp_core_affinitize(int cpu) 32 | { 33 | cpu_set_t cpus; 34 | struct bitmask *bmask; 35 | FILE *fp; 36 | char sysfname[MAX_FILE_NAME]; 37 | int phy_id; 38 | size_t n; 39 | int ret; 40 | int unused; 41 | 42 | n = GetNumCPUs(); 43 | 44 | if (cpu < 0 || cpu >= (int) n) { 45 | errno = -EINVAL; 46 | return -1; 47 | } 48 | 49 | CPU_ZERO(&cpus); 50 | CPU_SET((unsigned)cpu, &cpus); 51 | 52 | ret = sched_setaffinity(Gettid(), sizeof(cpus), &cpus); 53 | 54 | if (numa_max_node() == 0) 55 | return ret; 56 | 57 | bmask = numa_bitmask_alloc(numa_max_node() + 1); 58 | assert(bmask); 59 | 60 | /* read physical id of the core from sys information */ 61 | snprintf(sysfname, MAX_FILE_NAME - 1, 62 | "/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu); 63 | fp = fopen(sysfname, "r"); 64 | if (!fp) { 65 | perror(sysfname); 66 | errno = EFAULT; 67 | return -1; 68 | } 69 | ret = fscanf(fp, "%d", &phy_id); 70 | if (ret != 1) { 71 | fclose(fp); 72 | perror("Fail to read core id"); 73 | errno = EFAULT; 74 | return -1; 75 | } 76 | 77 | numa_bitmask_setbit(bmask, phy_id); 78 | numa_set_membind(bmask); 79 | numa_bitmask_free(bmask); 80 | 81 | fclose(fp); 82 | 83 | UNUSED(unused); 84 | return ret; 85 | } 86 | -------------------------------------------------------------------------------- /apps/epproxy/src/util.h: -------------------------------------------------------------------------------- 1 | #ifndef __UTIL_H_ 2 | #define __UTIL_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #define TRACE_CONFIG(f, m...) fprintf(stdout, f, ##m) 9 | 10 | #ifdef DBGERR 11 | 12 | #define TRACE_ERROR(f, m...) { \ 13 | fprintf(stdout, "[%10s:%4d] " f, __FUNCTION__, __LINE__, ##m); \ 14 | } 15 | 16 | #else 17 | 18 | #define TRACE_ERROR(f, m...) (void)0 19 | 20 | #endif /* DBGERR */ 21 | 22 | #ifdef DBGMSG 23 | 24 | #define TRACE_DBG(f, m...) {\ 25 | fprintf(stderr, "[%10s:%4d] " \ 26 | f, __FUNCTION__, __LINE__, ##m); \ 27 | } 28 | 29 | #else 30 | 31 | #define TRACE_DBG(f, m...) (void)0 32 | 33 | #endif /* DBGMSG */ 34 | 35 | #ifdef INFO 36 | 37 | #define TRACE_INFO(f, m...) { \ 38 | fprintf(stdout, "[%10s:%4d] " f,__FUNCTION__, __LINE__, ##m); \ 39 | } 40 | 41 | #else 42 | 43 | #define TRACE_INFO(f, m...) (void)0 44 | 45 | #endif /* INFO */ 46 | 47 | #ifdef EPOLL 48 | #define TRACE_EPOLL(f, m...) TRACE_FUNC("EPOLL", f, ##m) 49 | #else 50 | #define TRACE_EPOLL(f, m...) (void)0 51 | #endif 52 | 53 | #ifdef APP 54 | #define TRACE_APP(f, m...) TRACE_FUNC("APP", f, ##m) 55 | #else 56 | #define TRACE_APP(f, m...) (void)0 57 | #endif 58 | 59 | #ifdef DBGFUNC 60 | 61 | #define TRACE_FUNC(n, f, m...) { \ 62 | fprintf(stderr, "%6s: %10s:%4d] " \ 63 | f, n, __FUNCTION__, __LINE__, ##m); \ 64 | } 65 | 66 | #else 67 | 68 | #define TRACE_FUNC(f, m...) (void)0 69 | 70 | #endif /* DBGFUNC */ 71 | 72 | /*----------------------------------------------------------------------------*/ 73 | /* 74 | int 75 | mystrtol(const char *nptr, int base) 76 | { 77 | int rval; 78 | char *endptr; 79 | 80 | errno = 0; 81 | rval = strtol(nptr, &endptr, 10); 82 | if ((errno == ERANGE && (rval == LONG_MAX || 83 | rval == LONG_MIN)) 84 | || (errno != 0 && rval == 0)) { 85 | perror("strtol"); 86 | exit(EXIT_FAILURE); 87 | } 88 | if (endptr == nptr) { 89 | fprintf(stderr, "Parsing strtol error!\n"); 90 | exit(EXIT_FAILURE); 91 | } 92 | 93 | return rval; 94 | } 95 | */ 96 | #endif /* __UTIL_H_ */ 97 | -------------------------------------------------------------------------------- /mtcp/src/include/mtcp_epoll.h: -------------------------------------------------------------------------------- 1 | #ifndef __MTCP_EPOLL_H_ 2 | #define __MTCP_EPOLL_H_ 3 | 4 | #include "mtcp_api.h" 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | /*----------------------------------------------------------------------------*/ 11 | enum mtcp_epoll_op 12 | { 13 | MTCP_EPOLL_CTL_ADD = 1, 14 | MTCP_EPOLL_CTL_DEL = 2, 15 | MTCP_EPOLL_CTL_MOD = 3, 16 | }; 17 | /*----------------------------------------------------------------------------*/ 18 | enum mtcp_event_type 19 | { 20 | MTCP_EPOLLNONE = 0x000, 21 | MTCP_EPOLLIN = 0x001, 22 | MTCP_EPOLLPRI = 0x002, 23 | MTCP_EPOLLOUT = 0x004, 24 | MTCP_EPOLLRDNORM = 0x040, 25 | MTCP_EPOLLRDBAND = 0x080, 26 | MTCP_EPOLLWRNORM = 0x100, 27 | MTCP_EPOLLWRBAND = 0x200, 28 | MTCP_EPOLLMSG = 0x400, 29 | MTCP_EPOLLERR = 0x008, 30 | MTCP_EPOLLHUP = 0x010, 31 | MTCP_EPOLLRDHUP = 0x2000, 32 | MTCP_EPOLLONESHOT = (1 << 30), 33 | MTCP_EPOLLET = (1 << 31) 34 | }; 35 | /*----------------------------------------------------------------------------*/ 36 | typedef union mtcp_epoll_data 37 | { 38 | void *ptr; 39 | int sockid; 40 | uint32_t u32; 41 | uint64_t u64; 42 | } mtcp_epoll_data_t; 43 | /*----------------------------------------------------------------------------*/ 44 | struct mtcp_epoll_event 45 | { 46 | uint32_t events; 47 | mtcp_epoll_data_t data; 48 | }; 49 | /*----------------------------------------------------------------------------*/ 50 | int 51 | mtcp_epoll_create(mctx_t mctx, int size); 52 | /*----------------------------------------------------------------------------*/ 53 | int 54 | mtcp_epoll_ctl(mctx_t mctx, int epid, 55 | int op, int sockid, struct mtcp_epoll_event *event); 56 | /*----------------------------------------------------------------------------*/ 57 | int 58 | mtcp_epoll_wait(mctx_t mctx, int epid, 59 | struct mtcp_epoll_event *events, int maxevents, int timeout); 60 | /*----------------------------------------------------------------------------*/ 61 | char * 62 | EventToString(uint32_t event); 63 | /*----------------------------------------------------------------------------*/ 64 | 65 | #ifdef __cplusplus 66 | }; 67 | #endif 68 | 69 | #endif /* __MTCP_EPOLL_H_ */ 70 | -------------------------------------------------------------------------------- /apps/example/util.h: -------------------------------------------------------------------------------- 1 | #ifndef __UTIL_H_ 2 | #define __UTIL_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #define TRACE_CONFIG(f, m...) fprintf(stdout, f, ##m) 9 | 10 | #ifdef DBGERR 11 | 12 | #define TRACE_ERROR(f, m...) { \ 13 | fprintf(stdout, "[%10s:%4d] " f, __FUNCTION__, __LINE__, ##m); \ 14 | } 15 | 16 | #else 17 | 18 | #define TRACE_ERROR(f, m...) (void)0 19 | 20 | #endif /* DBGERR */ 21 | 22 | #ifdef DBGMSG 23 | 24 | #define TRACE_DBG(f, m...) {\ 25 | fprintf(stderr, "[%10s:%4d] " \ 26 | f, __FUNCTION__, __LINE__, ##m); \ 27 | } 28 | 29 | #else 30 | 31 | #define TRACE_DBG(f, m...) (void)0 32 | 33 | #endif /* DBGMSG */ 34 | 35 | #ifdef INFO 36 | 37 | #define TRACE_INFO(f, m...) { \ 38 | fprintf(stdout, "[%10s:%4d] " f,__FUNCTION__, __LINE__, ##m); \ 39 | } 40 | 41 | #else 42 | 43 | #define TRACE_INFO(f, m...) (void)0 44 | 45 | #endif /* INFO */ 46 | 47 | #ifdef EPOLL 48 | #define TRACE_EPOLL(f, m...) TRACE_FUNC("EPOLL", f, ##m) 49 | #else 50 | #define TRACE_EPOLL(f, m...) (void)0 51 | #endif 52 | 53 | #ifdef APP 54 | #define TRACE_APP(f, m...) TRACE_FUNC("APP", f, ##m) 55 | #else 56 | #define TRACE_APP(f, m...) (void)0 57 | #endif 58 | 59 | #ifdef DBGFUNC 60 | 61 | #define TRACE_FUNC(n, f, m...) { \ 62 | fprintf(stderr, "%6s: %10s:%4d] " \ 63 | f, n, __FUNCTION__, __LINE__, ##m); \ 64 | } 65 | 66 | #else 67 | 68 | #define TRACE_FUNC(f, m...) (void)0 69 | 70 | #endif /* DBGFUNC */ 71 | 72 | /*----------------------------------------------------------------------------*/ 73 | int 74 | mystrtol(const char *nptr, int base) 75 | { 76 | int rval; 77 | char *endptr; 78 | 79 | errno = 0; 80 | rval = strtol(nptr, &endptr, 10); 81 | /* check for strtol errors */ 82 | if ((errno == ERANGE && (rval == LONG_MAX || 83 | rval == LONG_MIN)) 84 | || (errno != 0 && rval == 0)) { 85 | perror("strtol"); 86 | exit(EXIT_FAILURE); 87 | } 88 | if (endptr == nptr) { 89 | fprintf(stderr, "Parsing strtol error!\n"); 90 | exit(EXIT_FAILURE); 91 | } 92 | 93 | return rval; 94 | } 95 | 96 | #endif /* __UTIL_H_ */ 97 | -------------------------------------------------------------------------------- /mtcp/src/intel_lthread/arch/x86/ctx.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * BSD LICENSE 3 | * 4 | * Copyright(c) 2015 Intel Corporation. All rights reserved. 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in 15 | * the documentation and/or other materials provided with the 16 | * distribution. 17 | * * Neither the name of Intel Corporation nor the names of its 18 | * contributors may be used to endorse or promote products derived 19 | * from this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | */ 33 | 34 | 35 | #ifndef CTX_H 36 | #define CTX_H 37 | 38 | /* 39 | * CPU context registers 40 | */ 41 | struct ctx { 42 | void *rsp; /* 0 */ 43 | void *rbp; /* 8 */ 44 | void *rip; /* 16 */ 45 | void *rbx; /* 24 */ 46 | void *r12; /* 32 */ 47 | void *r13; /* 40 */ 48 | void *r14; /* 48 */ 49 | void *r15; /* 56 */ 50 | }; 51 | 52 | 53 | void 54 | ctx_switch(struct ctx *new_ctx, struct ctx *curr_ctx); 55 | 56 | 57 | #endif /* RTE_CTX_H_ */ 58 | -------------------------------------------------------------------------------- /nic/setup_teardown/src/include/rss.h: -------------------------------------------------------------------------------- 1 | #ifndef RSS_H 2 | #define RSS_H 3 | /*------------------------------------------------------------------------*/ 4 | #include 5 | /*------------------------------------------------------------------------*/ 6 | #define BASE_PHY0_PORT_ID 0x0000 7 | #define BASE_VF0_PORT_ID 0x0300 8 | /*------------------------------------------------------------------------*/ 9 | /* RSS-specific variables */ 10 | /* RSS key that satisfies the condition for symmetric RSS 11 | (make sure that mTCP uses the same key to support GetRSSCPUCore()) */ 12 | __shared __declspec(local_mem) const uint8_t rss_key[HASH_TOEPLITZ_SECRET_KEY_SZ] = { 13 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 14 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 15 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 16 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 17 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05 18 | }; 19 | /*------------------------------------------------------------------------*/ 20 | /* RSS bitmask (NOTE: we consider (# cores) = (power of 2) case only) */ 21 | __shared __export __addr40 __imem uint8_t rss_bitmask; 22 | /*------------------------------------------------------------------------*/ 23 | uint32_t 24 | set_rss_bitmask(EXTRACTED_HEADERS_T *headers) { 25 | 26 | PIF_PLUGIN_ethernet_T *eth = pif_plugin_hdr_get_ethernet(headers); 27 | 28 | rss_bitmask = PIF_HEADER_GET_ethernet___srcAddr___0(eth); 29 | } 30 | /*------------------------------------------------------------------------*/ 31 | uint32_t 32 | get_rss_egress_port(EXTRACTED_HEADERS_T *headers) { 33 | 34 | PIF_PLUGIN_ipv4_T *ipv4 = pif_plugin_hdr_get_ipv4(headers); 35 | PIF_PLUGIN_tcp_T *tcp = pif_plugin_hdr_get_tcp(headers); 36 | uint32_t hash_key[3]; 37 | uint32_t hash_value; 38 | unsigned ingress_port; 39 | 40 | ingress_port = pif_plugin_meta_get__standard_metadata__ingress_port(headers); 41 | 42 | hash_key[0] = ipv4->srcAddr; 43 | hash_key[1] = ipv4->dstAddr; 44 | hash_key[2] = tcp->sdPorts; 45 | hash_value = hash_toeplitz((void *)hash_key, sizeof(hash_key), 46 | (void *)rss_key, HASH_TOEPLITZ_SECRET_KEY_SZ); 47 | 48 | return BASE_VF0_PORT_ID + (hash_value & rss_bitmask); 49 | } 50 | /*------------------------------------------------------------------------*/ 51 | #endif 52 | -------------------------------------------------------------------------------- /nic/setup_teardown/src/include/nfp_util.h: -------------------------------------------------------------------------------- 1 | #ifndef NFP_UTIL_H 2 | #define NFP_UTIL_H 3 | /*------------------------------------------------------------------------*/ 4 | #include 5 | /*------------------------------------------------------------------------*/ 6 | #define TS_1MS 0x124F8 7 | #define TS_5MS 0x5B8D8 8 | #define TS_10MS 0xB71B0 9 | #define TS_100MS 0x7270E0 10 | #define TS_500MS 0x23C30F0 11 | #define TS_1S 0x47868C0 12 | #define TS_5S 0x165A0BC0 13 | /*------------------------------------------------------------------------*/ 14 | uint64_t 15 | nfp_timer_get_cur_ts() 16 | { 17 | return (((uint64_t) local_csr_read(local_csr_timestamp_high) << 32) 18 | | local_csr_read(local_csr_timestamp_low)); 19 | } 20 | /*------------------------------------------------------------------------*/ 21 | uint32_t 22 | nfp_perform_crc32(uint32_t a, uint32_t b, uint32_t c) 23 | { 24 | uint32_t hash_key[3] = {a, b, c}; 25 | return hash_me_crc32((void *)hash_key, sizeof(hash_key), 1); 26 | } 27 | /*------------------------------------------------------------------------*/ 28 | /* note: should be used only in pif_plugin__from_host() */ 29 | int 30 | nfp_get_payloadlen() { 31 | return (pif_pkt_info_global.pkt_len - pif_pkt_info_global.pkt_pl_off); 32 | } 33 | /*------------------------------------------------------------------------*/ 34 | int 35 | nbi_get_payloadlen(PIF_PLUGIN_ipv4_T *ipv4) { 36 | return (ipv4->totalLen - TCPIP_HDRLEN); 37 | } 38 | /*------------------------------------------------------------------------*/ 39 | /* for debugging */ 40 | #define MILLION (1024*1024) 41 | __export __emem uint32_t wire_debug[MILLION]; 42 | __export __emem uint32_t wire_debug_idx; 43 | #define DEBUG(_a, _b, _c, _d) do { \ 44 | __xrw uint32_t _idx_val = 4; \ 45 | __xwrite uint32_t _dvals[4] = {_a, _b, _c, _d}; \ 46 | mem_test_add(&_idx_val, &wire_debug_idx, sizeof(_idx_val)); \ 47 | mem_write_atomic(_dvals, \ 48 | wire_debug + (_idx_val & (MILLION-1)), \ 49 | sizeof(_dvals)); \ 50 | } while(0) 51 | /*------------------------------------------------------------------------*/ 52 | #endif 53 | -------------------------------------------------------------------------------- /apps/epproxy/src/hash.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "hash.h" 6 | #include "epproxy.h" 7 | #include "util.h" 8 | /*----------------------------------------------------------------------------*/ 9 | struct backend_info* 10 | SelectNodeByHash(uint32_t hash) 11 | { 12 | struct backend_node* walk; 13 | int count = 0; 14 | 15 | if (TAILQ_EMPTY(&g_prx_ctx->bnode_hmap)) { 16 | TRACE_ERROR("should not happen!\n"); 17 | exit(-1); 18 | } 19 | 20 | TAILQ_FOREACH(walk, &g_prx_ctx->bnode_hmap, link) { 21 | count++; 22 | if (walk->hash > hash) 23 | return walk->binfo; 24 | } 25 | 26 | walk = TAILQ_FIRST(&g_prx_ctx->bnode_hmap); 27 | 28 | return walk->binfo; 29 | } 30 | /*----------------------------------------------------------------------------*/ 31 | void 32 | InsertHashNodes(struct backend_info* binfo) 33 | { 34 | struct backend_node* walk = NULL, *bnode; 35 | int j = 0; 36 | 37 | /* insert nodes in consistent hash ring */ 38 | for (j = 0; j < CHASH_WFACTOR_SCALE * binfo->weight; j++) { 39 | bnode = calloc(1, sizeof(struct backend_node)); 40 | bnode->binfo = binfo; 41 | 42 | bnode->hash = sdbm_hash(binfo->name, strlen(binfo->name)); 43 | bnode->hash += full_aval_hash(j); 44 | bnode->hash = full_aval_hash(bnode->hash); 45 | 46 | /* bnode_hmap is aligned by increasing order of hash values */ 47 | TAILQ_FOREACH(walk, &g_prx_ctx->bnode_hmap, link) { 48 | /* insert before the larger hash */ 49 | if (walk->hash > bnode->hash) { 50 | TAILQ_INSERT_BEFORE(walk, bnode, link); 51 | break; 52 | } 53 | } 54 | 55 | /* reached the end of the hash map 56 | (either it's empty or the given one has the largest value) */ 57 | if (!walk) 58 | TAILQ_INSERT_TAIL(&g_prx_ctx->bnode_hmap, bnode, link); 59 | } 60 | } 61 | /*----------------------------------------------------------------------------*/ 62 | int 63 | RemoveNodesByID(struct backend_info* binfo) 64 | { 65 | struct backend_node* walk; 66 | int res = -1; 67 | 68 | TAILQ_FOREACH(walk, &g_prx_ctx->bnode_hmap, link) { 69 | if (walk->binfo == binfo) { 70 | TAILQ_REMOVE(&g_prx_ctx->bnode_hmap, walk, link); 71 | res = 0; 72 | } 73 | } 74 | 75 | return res; 76 | } 77 | /*----------------------------------------------------------------------------*/ 78 | -------------------------------------------------------------------------------- /mtcp/src/include/icmp.h: -------------------------------------------------------------------------------- 1 | #ifndef __ICMP_H_ 2 | #define __ICMP_H_ 3 | /*----------------------------------------------------------------------------*/ 4 | struct icmphdr { 5 | uint8_t icmp_type; 6 | uint8_t icmp_code; 7 | uint16_t icmp_checksum; 8 | union { 9 | struct { 10 | uint16_t icmp_id; 11 | uint16_t icmp_sequence; 12 | } echo; // ECHO | ECHOREPLY 13 | struct { 14 | uint16_t unused; 15 | uint16_t nhop_mtu; 16 | } dest; // DEST_UNREACH 17 | } un; 18 | }; 19 | /*----------------------------------------------------------------------------*/ 20 | /* getters and setters for ICMP fields */ 21 | #define ICMP_ECHO_GET_ID(icmph) (icmph->un.echo.icmp_id) 22 | #define ICMP_ECHO_GET_SEQ(icmph) (icmph->un.echo.icmp_sequence) 23 | #define ICMP_DEST_UNREACH_GET_MTU(icmph) (icmph->un.dest.nhop_mtu) 24 | 25 | #define ICMP_ECHO_SET_ID(icmph, id) (icmph->un.echo.icmp_id = id) 26 | #define ICMP_ECHO_SET_SEQ(icmph, seq) (icmph->un.echo.icmp_sequence = seq) 27 | 28 | void 29 | RequestICMP(mtcp_manager_t mtcp, uint32_t saddr, uint32_t daddr, 30 | uint16_t icmp_id, uint16_t icmp_seq, 31 | uint8_t *icmpd, uint16_t len); 32 | 33 | int 34 | ProcessICMPPacket(mtcp_manager_t mtcp, struct iphdr *iph, int len); 35 | 36 | /* ICMP types */ 37 | #define ICMP_ECHOREPLY 0 /* Echo Reply */ 38 | #define ICMP_DEST_UNREACH 3 /* Destination Unreachable */ 39 | #define ICMP_SOURCE_QUENCH 4 /* Source Quench */ 40 | #define ICMP_REDIRECT 5 /* Redirect (change route) */ 41 | #define ICMP_ECHO 8 /* Echo Request */ 42 | #define ICMP_TIME_EXCEEDED 11 /* Time Exceeded */ 43 | #define ICMP_PARAMETERPROB 12 /* Parameter Problem */ 44 | #define ICMP_TIMESTAMP 13 /* Timestamp Request */ 45 | #define ICMP_TIMESTAMPREPLY 14 /* Timestamp Reply */ 46 | #define ICMP_INFO_REQUEST 15 /* Information Request */ 47 | #define ICMP_INFO_REPLY 16 /* Information Reply */ 48 | #define ICMP_ADDRESS 17 /* Address Mask Request */ 49 | #define ICMP_ADDRESSREPLY 18 /* Address Mask Reply */ 50 | /*----------------------------------------------------------------------------*/ 51 | #endif /* __ICMP_H_ */ 52 | -------------------------------------------------------------------------------- /mtcp/src/intel_lthread/lthread_mutex.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * BSD LICENSE 3 | * 4 | * Copyright(c) 2015 Intel Corporation. All rights reserved. 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in 15 | * the documentation and/or other materials provided with the 16 | * distribution. 17 | * * Neither the name of Intel Corporation nor the names of its 18 | * contributors may be used to endorse or promote products derived 19 | * from this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | */ 33 | 34 | 35 | #ifndef LTHREAD_MUTEX_H_ 36 | #define LTHREAD_MUTEX_H_ 37 | 38 | #include "lthread_queue.h" 39 | 40 | 41 | #define MAX_MUTEX_NAME_SIZE 64 42 | 43 | struct lthread_mutex { 44 | struct lthread *owner; 45 | rte_atomic64_t count; 46 | struct lthread_queue *blocked __rte_cache_aligned; 47 | struct lthread_sched *root_sched; 48 | char name[MAX_MUTEX_NAME_SIZE]; 49 | uint64_t diag_ref; /* optional ref to user diag data */ 50 | } __rte_cache_aligned; 51 | 52 | #endif /* LTHREAD_MUTEX_H_ */ 53 | -------------------------------------------------------------------------------- /apps/example/Makefile.in: -------------------------------------------------------------------------------- 1 | # TODO: Make this Makefile.in pretty 2 | 3 | TARGETS = msg_server msg_test 4 | CC=@CC@ -g -O3 -Wall -Werror -fgnu89-inline 5 | DPDK=@DPDK@ 6 | PS=@PSIO@ 7 | UCTX=@UCTX@ 8 | NETMAP=@NETMAP@ 9 | CFLAGS=@CFLAGS@ 10 | 11 | # DPDK LIBRARY and HEADER 12 | DPDK_INC=@DPDKLIBPATH@/include 13 | DPDK_LIB=@DPDKLIBPATH@/lib/ 14 | 15 | # mtcp library and header 16 | MTCP_FLD =../../mtcp/ 17 | MTCP_INC =-I${MTCP_FLD}/include 18 | MTCP_LIB =-L${MTCP_FLD}/lib 19 | MTCP_TARGET = ${MTCP_LIB}/libmtcp.a 20 | 21 | PS_DIR = ../../io_engine/ 22 | PS_INC = ${PS_DIR}/include 23 | INC = -I./include/ ${UTIL_INC} ${MTCP_INC} -I${UTIL_FLD}/include 24 | LIBS = ${MTCP_LIB} 25 | ifeq ($(PS),1) 26 | INC += -I{PS_INC} 27 | LIBS += -lmtcp -L${PS_DIR}/lib -lps -lpthread -lnuma -lrt 28 | endif 29 | 30 | ifeq ($(NETMAP),1) 31 | LIBS += -lmtcp -lpthread -lnuma -lrt 32 | endif 33 | 34 | # CFLAGS for DPDK-related compilation 35 | INC += ${MTCP_INC} 36 | ifeq ($(DPDK),1) 37 | DPDK_MACHINE_FLAGS = $(shell cat @DPDKLIBPATH@/include/cflags.txt) 38 | INC += ${DPDK_MACHINE_FLAGS} -I${DPDK_INC} -include $(DPDK_INC)/rte_config.h 39 | endif 40 | 41 | ifeq ($(shell uname -m),x86_64) 42 | LIBS += -m64 43 | endif 44 | 45 | ifeq ($(DPDK),1) 46 | DPDK_LIB_FLAGS = $(shell cat @DPDKLIBPATH@/lib/ldflags.txt) 47 | LIBS += -g -O3 -pthread -lrt -march=native -export-dynamic ${MTCP_FLD}/lib/libmtcp.a -L../../dpdk/lib -lnuma -lmtcp -lpthread -lrt -ldl ${DPDK_LIB_FLAGS} 48 | else 49 | LIBS += -g -O3 -pthread -lrt -march=native -export-dynamic ${MTCP_FLD}/lib/libmtcp.a -L../../dpdk/lib -lnuma -lmtcp -lpthread -lrt -ldl ${DPDK_LIB_FLAGS} 50 | endif 51 | 52 | ifeq ($(UCTX),1) 53 | CFLAGS += -DENABLE_UCTX 54 | endif 55 | ifeq ($V,) # no echo 56 | export MSG=@echo 57 | export HIDE=@ 58 | else 59 | export MSG=@\# 60 | export HIDE= 61 | endif 62 | 63 | all: msg_server msg_test 64 | 65 | msg_server.o: msg_server.c 66 | $(MSG) " CC $<" 67 | $(HIDE) ${CC} -c $< ${CFLAGS} ${INC} 68 | 69 | msg_server: msg_server.o ${MTCP_FLD}/lib/libmtcp.a 70 | $(MSG) " LD $<" 71 | $(HIDE) ${CC} $< ${LIBS} ${UTIL_OBJ} -o $@ 72 | 73 | msg_test.o: msg_test.c 74 | $(MSG) " CC $<" 75 | $(HIDE) ${CC} -c $< ${CFLAGS} ${INC} 76 | 77 | msg_test: msg_test.o ${MTCP_FLD}/lib/libmtcp.a 78 | $(MSG) " LD $<" 79 | $(HIDE) ${CC} $< ${LIBS} ${UTIL_OBJ} -o $@ 80 | 81 | clean: 82 | rm -f *~ *.o ${TARGETS} log_* 83 | 84 | distclean: clean 85 | rm -rf Makefile 86 | -------------------------------------------------------------------------------- /mtcp/src/intel_lthread/lthread_tls.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * BSD LICENSE 3 | * 4 | * Copyright(c) 2015 Intel Corporation. All rights reserved. 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in 15 | * the documentation and/or other materials provided with the 16 | * distribution. 17 | * * Neither the name of Intel Corporation nor the names of its 18 | * contributors may be used to endorse or promote products derived 19 | * from this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | */ 33 | 34 | #ifndef LTHREAD_TLS_H_ 35 | #define LTHREAD_TLS_H_ 36 | 37 | #include "lthread_api.h" 38 | 39 | #define RTE_PER_LTHREAD_SECTION_SIZE \ 40 | (&__stop_per_lt - &__start_per_lt) 41 | 42 | struct lthread_key { 43 | tls_destructor_func destructor; 44 | }; 45 | 46 | struct lthread_tls { 47 | void *data[LTHREAD_MAX_KEYS]; 48 | int nb_keys_inuse; 49 | struct lthread_sched *root_sched; 50 | }; 51 | 52 | void _lthread_tls_destroy(struct lthread *lt); 53 | void _lthread_key_pool_init(void); 54 | void _lthread_tls_alloc(struct lthread *lt); 55 | 56 | 57 | #endif /* LTHREAD_TLS_H_ */ 58 | -------------------------------------------------------------------------------- /mtcp/src/include/tcp_out.h: -------------------------------------------------------------------------------- 1 | #ifndef __TCP_OUT_H_ 2 | #define __TCP_OUT_H_ 3 | 4 | #include "mtcp.h" 5 | #include "tcp_stream.h" 6 | 7 | enum ack_opt 8 | { 9 | ACK_OPT_NOW, 10 | ACK_OPT_AGGREGATE, 11 | ACK_OPT_WACK 12 | }; 13 | 14 | uint8_t* 15 | SendTCPPacketStandalone(struct mtcp_manager *mtcp, 16 | uint32_t saddr, uint16_t sport, uint32_t daddr, uint16_t dport, 17 | uint32_t seq, uint32_t ack_seq, uint16_t window, uint8_t flags, 18 | const uint8_t *payload, uint16_t payloadlen, 19 | uint32_t cur_ts, uint32_t echo_ts); 20 | 21 | int 22 | SendTCPPacket(struct mtcp_manager *mtcp, tcp_stream *cur_stream, 23 | uint32_t cur_ts, uint8_t flags, uint8_t *payload, uint16_t payloadlen); 24 | 25 | uint8_t* 26 | SendTCPOffloadPacketStandalone(struct mtcp_manager *mtcp, uint16_t offload_type, 27 | uint32_t saddr, uint16_t sport, uint32_t daddr, uint16_t dport, 28 | uint32_t seq, uint32_t ack_seq, uint16_t window, uint8_t flags, 29 | const uint8_t *payload, uint16_t payloadlen, 30 | uint32_t cur_ts, uint32_t echo_ts); 31 | 32 | int 33 | SendTCPOffloadPacket(struct mtcp_manager *mtcp, tcp_stream *cur_stream, 34 | uint32_t cur_ts, uint8_t flags, uint8_t *payload, uint16_t payloadlen, 35 | struct mtcp_offload_meta* offload_meta); 36 | 37 | extern inline int 38 | WriteTCPControlList(mtcp_manager_t mtcp, 39 | struct mtcp_sender *sender, uint32_t cur_ts, int thresh); 40 | 41 | extern inline int 42 | WriteTCPDataList(mtcp_manager_t mtcp, 43 | struct mtcp_sender *sender, uint32_t cur_ts, int thresh); 44 | 45 | extern inline int 46 | WriteTCPACKList(mtcp_manager_t mtcp, 47 | struct mtcp_sender *sender, uint32_t cur_ts, int thresh); 48 | 49 | extern inline void 50 | AddtoControlList(mtcp_manager_t mtcp, tcp_stream *cur_stream, uint32_t cur_ts); 51 | 52 | extern inline void 53 | AddtoSendList(mtcp_manager_t mtcp, tcp_stream *cur_stream); 54 | 55 | extern inline void 56 | RemoveFromControlList(mtcp_manager_t mtcp, tcp_stream *cur_stream); 57 | 58 | extern inline void 59 | RemoveFromSendList(mtcp_manager_t mtcp, tcp_stream *cur_stream); 60 | 61 | extern inline void 62 | RemoveFromACKList(mtcp_manager_t mtcp, tcp_stream *cur_stream); 63 | 64 | extern inline void 65 | EnqueueACK(mtcp_manager_t mtcp, 66 | tcp_stream *cur_stream, uint32_t cur_ts, uint8_t opt); 67 | 68 | extern inline void 69 | DumpControlList(mtcp_manager_t mtcp, struct mtcp_sender *sender); 70 | 71 | #if TCP_CALCULATE_CHECKSUM 72 | #ifdef DISABLE_HWCSUM 73 | void 74 | UpdateTCPChecksum(struct tcphdr* tcph, tcp_stream *cur_stream); 75 | #endif 76 | #endif 77 | 78 | #endif /* __TCP_OUT_H_ */ 79 | -------------------------------------------------------------------------------- /mtcp/src/intel_lthread/lthread_timer.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * BSD LICENSE 3 | * 4 | * Copyright(c) 2015 Intel Corporation. All rights reserved. 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in 15 | * the documentation and/or other materials provided with the 16 | * distribution. 17 | * * Neither the name of Intel Corporation nor the names of its 18 | * contributors may be used to endorse or promote products derived 19 | * from this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | */ 33 | 34 | 35 | #ifndef LTHREAD_TIMER_H_ 36 | #define LTHREAD_TIMER_H_ 37 | 38 | #include "lthread_int.h" 39 | #include "lthread_sched.h" 40 | 41 | 42 | static inline uint64_t 43 | _ns_to_clks(uint64_t ns) 44 | { 45 | unsigned __int128 clkns = rte_get_tsc_hz(); 46 | 47 | clkns *= ns; 48 | clkns /= 1000000000; 49 | return (uint64_t) clkns; 50 | } 51 | 52 | 53 | static inline void 54 | _timer_start(struct lthread *lt, uint64_t clks) 55 | { 56 | if (clks > 0) { 57 | DIAG_EVENT(lt, LT_DIAG_LTHREAD_TMR_START, <->tim, clks); 58 | rte_timer_init(<->tim); 59 | rte_timer_reset(<->tim, 60 | clks, 61 | SINGLE, 62 | rte_lcore_id(), 63 | _sched_timer_cb, 64 | (void *)lt); 65 | } 66 | } 67 | 68 | 69 | static inline void 70 | _timer_stop(struct lthread *lt) 71 | { 72 | if (lt != NULL) { 73 | DIAG_EVENT(lt, LT_DIAG_LTHREAD_TMR_DELETE, <->tim, 0); 74 | rte_timer_stop(<->tim); 75 | } 76 | } 77 | 78 | 79 | #endif /* LTHREAD_TIMER_H_ */ 80 | -------------------------------------------------------------------------------- /mtcp/src/include/ip_in.h: -------------------------------------------------------------------------------- 1 | #ifndef __IP_IN_H_ 2 | #define __IP_IN_H_ 3 | 4 | #include "mtcp.h" 5 | 6 | #ifdef DISABLE_PSIO 7 | #include 8 | #if defined(__i386__) || defined(__x86_64__) 9 | static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) 10 | { 11 | unsigned int sum; 12 | 13 | asm(" movl (%1), %0\n" 14 | " subl $4, %2\n" 15 | " jbe 2f\n" 16 | " addl 4(%1), %0\n" 17 | " adcl 8(%1), %0\n" 18 | " adcl 12(%1), %0\n" 19 | "1: adcl 16(%1), %0\n" 20 | " lea 4(%1), %1\n" 21 | " decl %2\n" 22 | " jne 1b\n" 23 | " adcl $0, %0\n" 24 | " movl %0, %2\n" 25 | " shrl $16, %0\n" 26 | " addw %w2, %w0\n" 27 | " adcl $0, %0\n" 28 | " notl %0\n" 29 | "2:" 30 | /* Since the input registers which are loaded with iph and ih 31 | are modified, we must also specify them as outputs, or gcc 32 | will assume they contain their original values. */ 33 | : "=r" (sum), "=r" (iph), "=r" (ihl) 34 | : "1" (iph), "2" (ihl) 35 | : "memory"); 36 | return (__sum16)sum; 37 | } 38 | #else 39 | #define __force 40 | typedef unsigned int u32; 41 | 42 | static inline __sum16 csum_fold(__wsum csum) 43 | { 44 | u32 sum = (__force u32)csum;; 45 | 46 | sum += (sum << 16); 47 | csum = (sum < csum); 48 | sum >>= 16; 49 | sum += csum; 50 | 51 | return (__force __sum16)~sum; 52 | } 53 | 54 | /* 55 | * This is a version of ip_compute_csum() optimized for IP headers, 56 | * which always checksum on 4 octet boundaries. 57 | * 58 | * By Jorge Cwik , adapted for linux by 59 | * Arnt Gulbrandsen. 60 | */ 61 | static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) 62 | { 63 | const unsigned int *word = iph; 64 | const unsigned int *stop = word + ihl; 65 | unsigned int csum; 66 | int carry; 67 | 68 | csum = word[0]; 69 | csum += word[1]; 70 | carry = (csum < word[1]); 71 | csum += carry; 72 | 73 | csum += word[2]; 74 | carry = (csum < word[2]); 75 | csum += carry; 76 | 77 | csum += word[3]; 78 | carry = (csum < word[3]); 79 | csum += carry; 80 | 81 | word += 4; 82 | do { 83 | csum += *word; 84 | carry = (csum < *word); 85 | csum += carry; 86 | word++; 87 | } while (word != stop); 88 | 89 | return csum_fold(csum); 90 | } 91 | #endif /* defined(__i386__) || defined(__x86_64__) */ 92 | #endif /* DISABLE_PSIO */ 93 | 94 | int 95 | ProcessIPv4Packet(mtcp_manager_t mtcp, uint32_t cur_ts, 96 | const int ifidx, unsigned char* pkt_data, int len); 97 | inline int 98 | ProcessSetupOffloadPacket(mtcp_manager_t mtcp, uint32_t cur_ts, 99 | const int ifidx, unsigned char* pkt_data, int len); 100 | #endif /* __IP_IN_H_ */ 101 | -------------------------------------------------------------------------------- /nic/splice/src/include/rss.h: -------------------------------------------------------------------------------- 1 | #ifndef RSS_H 2 | #define RSS_H 3 | /*------------------------------------------------------------------------*/ 4 | /* RSS-specific variables */ 5 | 6 | /* the number of LSBs taken from the hash value 7 | (e.g., 9 LSBs in Intel 40GbE NIC, see Section 7.1.8 of XL710-QDA2 datasheet) */ 8 | #define RSS_LSB_MASK 0x01FF 9 | 10 | /* RSS key that satisfies the condition for symmetric RSS 11 | (make sure that mTCP uses the same key to support GetRSSCPUCore()) */ 12 | __shared __declspec(local_mem) const uint8_t rss_key[HASH_TOEPLITZ_SECRET_KEY_SZ] = { 13 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 14 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 15 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 16 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 17 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05 18 | }; 19 | /*------------------------------------------------------------------------*/ 20 | /* RSS bitmask (NOTE: we consider (# cores) = (power of 2) case only) */ 21 | __shared __export volatile __addr40 __imem uint8_t rss_bitmask = 0; 22 | /*------------------------------------------------------------------------*/ 23 | void 24 | set_rss_bitmask(EXTRACTED_HEADERS_T *headers) { 25 | 26 | PIF_PLUGIN_ethernet_T *eth = pif_plugin_hdr_get_ethernet(headers); 27 | 28 | rss_bitmask = PIF_HEADER_GET_ethernet___srcAddr___0(eth); 29 | } 30 | /*------------------------------------------------------------------------*/ 31 | uint32_t 32 | get_rss_egress_port(EXTRACTED_HEADERS_T *headers) { 33 | 34 | PIF_PLUGIN_ipv4_T *ipv4 = pif_plugin_hdr_get_ipv4(headers); 35 | PIF_PLUGIN_tcp_T *tcp = pif_plugin_hdr_get_tcp(headers); 36 | 37 | return get_rss_egress_port_by_value( 38 | pif_plugin_meta_get__standard_metadata__ingress_port(headers), 39 | ipv4->srcAddr, 40 | ipv4->dstAddr, 41 | tcp->sdPorts); 42 | } 43 | /*------------------------------------------------------------------------*/ 44 | uint32_t 45 | get_rss_egress_port_by_value(uint32_t ingress_port, uint32_t srcAddr, 46 | uint32_t dstAddr, uint32_t sdPorts) { 47 | 48 | /* For FIN management, always send to BASE_VF0 */ 49 | volatile uint32_t hash_key[3]; 50 | uint32_t hash_value; 51 | uint32_t egress_port; 52 | 53 | hash_key[0] = srcAddr; 54 | hash_key[1] = dstAddr; 55 | hash_key[2] = sdPorts; 56 | hash_value = hash_toeplitz((void *)hash_key, 57 | sizeof(hash_key), 58 | (void *)rss_key, 59 | HASH_TOEPLITZ_SECRET_KEY_SZ); 60 | 61 | egress_port = (ingress_port == BASE_PHY0_PORT_ID) ? 62 | BASE_VF0_PORT_ID : BASE_VF1_PORT_ID; 63 | egress_port += (hash_value & rss_bitmask); 64 | return egress_port; 65 | } 66 | /*------------------------------------------------------------------------*/ 67 | #endif 68 | 69 | -------------------------------------------------------------------------------- /apps/epproxy/src/backend_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef _INCLUDE_BACKEND_POOL_H_ 2 | #define _INCLUDE_BACKEND_POOL_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | /* use pthread_rwlock by default (rte_rwlock should be tested further) */ 12 | #define USE_PTHREAD_RWLOCK 13 | 14 | #ifdef USE_PTHREAD_RWLOCK 15 | #include 16 | #else 17 | #include 18 | #endif /* USE_PTHREAD_RWLOCK */ 19 | 20 | //#include "util.h" 21 | 22 | /* 23 | enum addr_type { 24 | LB_BACKEND_IPV4, 25 | LB_BACKEND_IPV6 26 | }; 27 | */ 28 | 29 | typedef struct backend_info { 30 | struct sockaddr_in addr; /* backend address (IPv4) */ 31 | char* name; /* backend name */ 32 | int weight; /* backend server load weight (C-HASH) */ 33 | 34 | TAILQ_ENTRY (backend_info) link; /* backend info link */ 35 | 36 | /* list of free persistent connections to backend servers */ 37 | TAILQ_HEAD (, http_stream) idle_conns[16]; /* FIXME: fixed number */ 38 | 39 | /* a connection for health check to backend servers */ 40 | struct http_stream *hc_conn; 41 | 42 | /* those fields can be used in the future */ 43 | // enum hc_state bhealth; /* backend health */ 44 | 45 | uint8_t addr_done:1, 46 | name_done:1, 47 | weight_done:1; 48 | 49 | } backend_info; 50 | 51 | typedef struct backend_pool { 52 | size_t pool_size; /* size of backend pool */ 53 | TAILQ_HEAD(, backend_info) bi_list; /* backend info list */ 54 | 55 | #ifdef USE_PTHREAD_RWLOCK 56 | pthread_rwlock_t pool_lock; 57 | #else 58 | rte_rwlock_t pool_lock; 59 | #endif 60 | 61 | /* those fields can be used in the future */ 62 | // enum hc_method hcm; /* health check method for this pool */ 63 | // bool dsr; /* DSR-based pool */ 64 | // enum addr_type atype; /* IP address type (IPv4 or IPv6) */ 65 | 66 | } backend_pool; 67 | 68 | //struct backend_pool active_pool[MAX_CPUS]; 69 | 70 | /* initialize and destroy pool */ 71 | int init_backend_pool(struct backend_pool * 72 | /*, enum hc_method, enum addr_type, bool dsr*/); 73 | void destory_backend_pool(struct backend_pool *); 74 | 75 | /* add and remove backend */ 76 | int add_to_bpool(struct backend_pool *, struct sockaddr_in *, char *, int); 77 | int add_to_bpool_entry(struct backend_pool *pool, struct backend_info* back_ctx); 78 | int remove_from_bpool_by_addr(struct backend_pool *, struct sockaddr_in *); 79 | int remove_from_bpool_by_pos(struct backend_pool *, size_t); 80 | 81 | /* get information from pool */ 82 | size_t get_backend_poolsize(struct backend_pool *); 83 | struct backend_info *get_server_from_bpool_by_addr(struct backend_pool *, struct sockaddr_in *); 84 | struct backend_info *get_server_from_bpool_by_pos(struct backend_pool *, size_t); 85 | 86 | #endif /* _INCLUDE_BACKEND_POOL_H_ */ 87 | -------------------------------------------------------------------------------- /mtcp/src/ip_in.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "ip_in.h" 5 | #include "tcp_in.h" 6 | #include "mtcp_api.h" 7 | #ifndef DISABLE_PSIO 8 | #include "ps.h" 9 | #endif 10 | #include "debug.h" 11 | #include "icmp.h" 12 | 13 | #define ETH_P_IP_FRAG 0xF800 14 | #define ETH_P_IPV6_FRAG 0xF6DD 15 | 16 | /*----------------------------------------------------------------------------*/ 17 | inline int 18 | ProcessIPv4Packet(mtcp_manager_t mtcp, uint32_t cur_ts, 19 | const int ifidx, unsigned char* pkt_data, int len) 20 | { 21 | /* check and process IPv4 packets */ 22 | struct iphdr* iph = (struct iphdr *)(pkt_data + sizeof(struct ethhdr)); 23 | int ip_len = ntohs(iph->tot_len); 24 | int rc = -1; 25 | 26 | /* drop the packet shorter than ip header */ 27 | if (ip_len < sizeof(struct iphdr)) { 28 | return ERROR; 29 | } 30 | 31 | #ifndef DISABLE_HWCSUM 32 | if (mtcp->iom->dev_ioctl != NULL) 33 | rc = mtcp->iom->dev_ioctl(mtcp->ctx, ifidx, PKT_RX_IP_CSUM, iph); 34 | if (rc == -1 && ip_fast_csum(iph, iph->ihl)) 35 | return ERROR; 36 | #else 37 | UNUSED(rc); 38 | if (ip_fast_csum(iph, iph->ihl)){ 39 | return ERROR; 40 | } 41 | #endif 42 | 43 | #if !PROMISCUOUS_MODE 44 | /* if not promiscuous mode, drop if the destination is not myself */ 45 | if (iph->daddr != CONFIG.eths[ifidx].ip_addr) 46 | //DumpIPPacketToFile(stderr, iph, ip_len); 47 | return TRUE; 48 | #endif 49 | 50 | // see if the version is correct 51 | if (iph->version != 0x4 ) { 52 | mtcp->iom->release_pkt(mtcp->ctx, ifidx, pkt_data, len); 53 | return FALSE; 54 | } 55 | 56 | switch (iph->protocol) { 57 | case IPPROTO_TCP: 58 | return ProcessTCPPacket(mtcp, cur_ts, ifidx, iph, ip_len); 59 | case IPPROTO_ICMP: 60 | return ProcessICMPPacket(mtcp, iph, ip_len); 61 | default: 62 | /* currently drop other protocols */ 63 | return FALSE; 64 | } 65 | return FALSE; 66 | } 67 | /*----------------------------------------------------------------------------*/ 68 | inline int 69 | ProcessSetupOffloadPacket(mtcp_manager_t mtcp, uint32_t cur_ts, 70 | const int ifidx, unsigned char* pkt_data, int len) 71 | { 72 | /* check and process IPv4 packets */ 73 | struct iphdr* iph = (struct iphdr *)(pkt_data + sizeof(struct ethhdr)); 74 | int ip_len = ntohs(iph->tot_len); 75 | int rc = -1; 76 | 77 | /* drop the packet shorter than ip header */ 78 | if (ip_len < sizeof(struct iphdr)) 79 | return ERROR; 80 | 81 | #ifndef DISABLE_HWCSUM 82 | if (mtcp->iom->dev_ioctl != NULL) 83 | rc = mtcp->iom->dev_ioctl(mtcp->ctx, ifidx, PKT_RX_IP_CSUM, iph); 84 | if (rc == -1 && ip_fast_csum(iph, iph->ihl)) 85 | return ERROR; 86 | #else 87 | UNUSED(rc); 88 | if (ip_fast_csum(iph, iph->ihl)) 89 | return ERROR; 90 | #endif 91 | return ProcessCUSTOMPacket(mtcp, cur_ts, ifidx, iph, ip_len); 92 | } 93 | /*----------------------------------------------------------------------------*/ 94 | -------------------------------------------------------------------------------- /nic/splice/src/include/common.h: -------------------------------------------------------------------------------- 1 | #ifndef COMMON_H 2 | #define COMMON_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | /*------------------------------------------------------------------------*/ 12 | /* Enable Connection Setup Offload */ 13 | #define ENABLE_SETUP_OFFLOAD 1 14 | /*------------------------------------------------------------------------*/ 15 | /* Update Checksum */ 16 | #define UPDATE_CHECKSUM 1 17 | /*------------------------------------------------------------------------*/ 18 | /* NIC MAC Address */ 19 | 20 | /* Port 0 */ 21 | #define PORT0_MAC_0 0x00154d12 22 | #define PORT0_MAC_1 0x129b 23 | 24 | /* Port 1 */ 25 | #define PORT1_MAC_0 0x00154d12 26 | #define PORT1_MAC_1 0x129c 27 | /*------------------------------------------------------------------------*/ 28 | /* For RSS */ 29 | #define NUM_RX_CORES 8 30 | #define ETHERTYPE_OFFLOAD_INIT_CTRL 0x080C 31 | #define ETHERTYPE_SETUP_OFFLOAD 0x0809 32 | 33 | #define BASE_PHY0_PORT_ID 0x0000 34 | #define BASE_PHY1_PORT_ID 0x0004 35 | #define BASE_VF0_PORT_ID 0x0300 36 | #define BASE_VF1_PORT_ID (0x0300 + NUM_RX_CORES) 37 | /*------------------------------------------------------------------------*/ 38 | /* For Packet Management */ 39 | #define CTRL_P_SPLICE_FINISH 0x0901 40 | #define TCP_DATA_LEN 10 41 | #define SWAP(ports) ((ports >>16) | (ports << 16)) 42 | /*------------------------------------------------------------------------*/ 43 | /* TCP Flag Value */ 44 | #define TCP_FLAG_FIN 0x01 // 0000 0001 45 | #define TCP_FLAG_SYN 0x02 // 0000 0010 46 | #define TCP_FLAG_RST 0x04 // 0000 0100 47 | #define TCP_FLAG_PSH 0x08 // 0000 1000 48 | #define TCP_FLAG_ACK 0x10 // 0001 0000 49 | #define TCP_FLAG_URG 0x20 // 0010 0000 50 | /*------------------------------------------------------------------------*/ 51 | #define PACKET_CASE_NORMAL 0x00 // 0000 52 | #define PACKET_CASE_RST 0x01 // 0001 53 | #define PACKET_CASE_FIN 0x02 // 0010 54 | #define PACKET_CASE_FINACK 0x04 // 0100 55 | /*------------------------------------------------------------------------*/ 56 | /* for debugging */ 57 | __export __emem uint32_t wire_debug[1024*1024]; 58 | __export __emem uint32_t wire_debug_idx; 59 | #define DEBUG(_a, _b, _c, _d) do { \ 60 | __xrw uint32_t _idx_val = 4; \ 61 | __xwrite uint32_t _dvals[4]; \ 62 | mem_test_add(&_idx_val, &wire_debug_idx, sizeof(_idx_val)); \ 63 | _dvals[0] = _a; \ 64 | _dvals[1] = _b; \ 65 | _dvals[2] = _c; \ 66 | _dvals[3] = _d; \ 67 | mem_write_atomic(_dvals, \ 68 | wire_debug + (_idx_val % (1024 * 1024)), sizeof(_dvals)); \ 69 | } while(0) 70 | /*------------------------------------------------------------------------*/ 71 | #endif /* COMMON_H */ 72 | -------------------------------------------------------------------------------- /mtcp/src/include/tcp_stream_queue.h: -------------------------------------------------------------------------------- 1 | #ifndef __TCP_STREAM_QUEUE_ 2 | #define __TCP_STREAM_QUEUE_ 3 | 4 | #include 5 | 6 | /* Lock definitions for stream queue */ 7 | #if LOCK_STREAM_QUEUE 8 | 9 | #if USE_SPIN_LOCK 10 | #define SQ_LOCK_INIT(lock, errmsg, action); \ 11 | if (pthread_spin_init(lock, PTHREAD_PROCESS_PRIVATE)) { \ 12 | perror("pthread_spin_init" errmsg); \ 13 | action; \ 14 | } 15 | #define SQ_LOCK_DESTROY(lock) pthread_spin_destroy(lock) 16 | #define SQ_LOCK(lock) pthread_spin_lock(lock) 17 | #define SQ_UNLOCK(lock) pthread_spin_unlock(lock) 18 | #else 19 | #define SQ_LOCK_INIT(lock, errmsg, action); \ 20 | if (pthread_mutex_init(lock, NULL)) { \ 21 | perror("pthread_mutex_init" errmsg); \ 22 | action; \ 23 | } 24 | #define SQ_LOCK_DESTROY(lock) pthread_mutex_destroy(lock) 25 | #define SQ_LOCK(lock) pthread_mutex_lock(lock) 26 | #define SQ_UNLOCK(lock) pthread_mutex_unlock(lock) 27 | #endif /* USE_SPIN_LOCK */ 28 | 29 | #else /* LOCK_STREAM_QUEUE */ 30 | #define SQ_LOCK_INIT(lock, errmsg, action) (void) 0 31 | #define SQ_LOCK_DESTROY(lock) (void) 0 32 | #define SQ_LOCK(lock) (void) 0 33 | #define SQ_UNLOCK(lock) (void) 0 34 | #endif /* LOCK_STREAM_QUEUE */ 35 | 36 | /*---------------------------------------------------------------------------*/ 37 | typedef struct stream_queue* stream_queue_t; 38 | /*---------------------------------------------------------------------------*/ 39 | typedef struct stream_queue_int 40 | { 41 | struct tcp_stream **array; 42 | int size; 43 | 44 | int first; 45 | int last; 46 | int count; 47 | 48 | } stream_queue_int; 49 | /*---------------------------------------------------------------------------*/ 50 | stream_queue_int * 51 | CreateInternalStreamQueue(int size); 52 | /*---------------------------------------------------------------------------*/ 53 | void 54 | DestroyInternalStreamQueue(stream_queue_int *sq); 55 | /*---------------------------------------------------------------------------*/ 56 | int 57 | StreamInternalEnqueue(stream_queue_int *sq, struct tcp_stream *stream); 58 | /*---------------------------------------------------------------------------*/ 59 | struct tcp_stream * 60 | StreamInternalDequeue(stream_queue_int *sq); 61 | /*---------------------------------------------------------------------------*/ 62 | stream_queue_t 63 | CreateStreamQueue(int size); 64 | /*---------------------------------------------------------------------------*/ 65 | void 66 | DestroyStreamQueue(stream_queue_t sq); 67 | /*---------------------------------------------------------------------------*/ 68 | int 69 | StreamEnqueue(stream_queue_t sq, struct tcp_stream *stream); 70 | /*---------------------------------------------------------------------------*/ 71 | struct tcp_stream * 72 | StreamDequeue(stream_queue_t sq); 73 | /*---------------------------------------------------------------------------*/ 74 | int 75 | StreamQueueIsEmpty(stream_queue_t sq); 76 | /*---------------------------------------------------------------------------*/ 77 | 78 | #endif /* __TCP_STREAM_QUEUE_ */ 79 | -------------------------------------------------------------------------------- /mtcp/src/nic_control.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "mtcp.h" 5 | #include "nic_control.h" 6 | #include "eth_out.h" 7 | #include "debug.h" 8 | 9 | #include "splice_pool.h" 10 | #include "ip_out.h" 11 | 12 | struct control_splice_finish_hdr 13 | { 14 | uint16_t close_reason; /* the reason of spliced connection close */ 15 | uint32_t ip_addr; /* the ip address of client-side spliced connection */ 16 | uint16_t port; /* the port of client-side spliced connection */ 17 | uint32_t unused; /* unused field */ 18 | uint16_t eth_type; /* the ethernet type to distinguish the type of the packet */ 19 | uint32_t dst_addr; 20 | uint16_t dst_port; 21 | } __attribute__ ((packed)); 22 | 23 | void 24 | DumpSpliceFinishPacket(struct control_splice_finish_hdr *ctrlh); 25 | 26 | int 27 | ProcessSpliceFinishPacket(mtcp_manager_t mtcp, uint32_t cur_ts, 28 | const int ifidx, unsigned char *pkt_data, int len) 29 | { 30 | struct control_splice_finish_hdr* ctrlh = 31 | (struct control_splice_finish_hdr* )pkt_data; 32 | struct sockaddr_in splice_addr; 33 | int ret; 34 | 35 | struct addr_pool *ap_walk = NULL; 36 | struct addr_pool *splice_ap = NULL; 37 | 38 | #ifdef VERBOSE 39 | DumpSpliceFinishPacket(ctrlh); 40 | #endif 41 | splice_addr.sin_addr.s_addr = ctrlh->ip_addr; 42 | splice_addr.sin_port = ctrlh->port; 43 | 44 | TAILQ_FOREACH(ap_walk, &mtcp->ap_list, ap_link) { 45 | if (ap_walk->daddr == ctrlh->dst_addr && ap_walk->dport == ctrlh->dst_port) { 46 | splice_ap = ap_walk; 47 | break; 48 | } 49 | } 50 | if (splice_ap) { 51 | ret = FreeSpliceAddress(splice_ap, &splice_addr); 52 | } else { 53 | uint8_t is_external; 54 | int nif = GetOutputInterface(splice_addr.sin_addr.s_addr, &is_external); 55 | if (nif < 0) { 56 | TRACE_ERROR("nif is negative!\n"); 57 | ret = -1; 58 | } else { 59 | int eidx = CONFIG.nif_to_eidx[nif]; 60 | ret = FreeSpliceAddress(ap[eidx], &splice_addr); 61 | } 62 | UNUSED(is_external); 63 | } 64 | if (ret < 0) { 65 | TRACE_ERROR("(NEVER HAPPEN) Failed to free address.\n"); 66 | } 67 | else { 68 | if (mtcp->cb) { 69 | nsplice_meta_t meta; 70 | meta.ip_addr = ctrlh->ip_addr; 71 | meta.port = ctrlh->port; 72 | meta.dst_addr = ctrlh->dst_addr; 73 | meta.dst_port = ctrlh->dst_port; 74 | meta.close_reason = ctrlh->close_reason; 75 | mtcp->cb(&meta); 76 | } 77 | } 78 | 79 | 80 | return ret; 81 | } 82 | 83 | void 84 | DumpSpliceFinishPacket(struct control_splice_finish_hdr *ctrlh) 85 | { 86 | TRACE_INFO("--------------------------------\n" 87 | "Splice Finished Packet\n" 88 | "close_reason: %X\n" 89 | "ip_addr: %x\n" 90 | "port: %u\n" 91 | "control_type: %X\n" 92 | "dst_addr: %x\n" 93 | "dst_port: %u\n" 94 | "\n", 95 | ntohs(ctrlh->close_reason), 96 | ntohl(ctrlh->ip_addr), 97 | ntohs(ctrlh->port), 98 | ntohs(ctrlh->eth_type), 99 | ntohl(ctrlh->dst_addr), 100 | ntohs(ctrlh->dst_port)); 101 | } 102 | /*----------------------------------------------------------------------------*/ 103 | -------------------------------------------------------------------------------- /mtcp/src/include/tcp_ring_buffer.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * 2010.12.10 Shinae Woo 4 | * Ring buffer structure for managing dynamically allocating ring buffer 5 | * 6 | * put data to the tail 7 | * get/pop/remove data from the head 8 | * 9 | * always garantee physically continuous ready in-memory data from data_offset to the data_offset+len 10 | * automatically increase total buffer size when buffer is full 11 | * for efficiently managing packet payload and chunking 12 | * 13 | */ 14 | 15 | #ifndef __NRE_RING_BUFFER_ 16 | #define __NRE_RING_BUFFER_ 17 | 18 | #include 19 | #include 20 | 21 | /*----------------------------------------------------------------------------*/ 22 | enum rb_caller 23 | { 24 | AT_APP, 25 | AT_MTCP 26 | }; 27 | /*----------------------------------------------------------------------------*/ 28 | #ifdef ENABLELRO 29 | typedef struct mtcp_manager* mtcp_manager_t; 30 | #endif 31 | typedef struct rb_manager* rb_manager_t; 32 | /*----------------------------------------------------------------------------*/ 33 | struct fragment_ctx 34 | { 35 | uint32_t seq; 36 | uint32_t len : 31; 37 | uint32_t is_calloc : 1; 38 | struct fragment_ctx *next; 39 | }; 40 | /*----------------------------------------------------------------------------*/ 41 | struct tcp_ring_buffer 42 | { 43 | u_char* data; /* buffered data */ 44 | u_char* head; /* pointer to the head */ 45 | 46 | uint32_t head_offset; /* offset for the head (head - data) */ 47 | uint32_t tail_offset; /* offset fot the last byte (null byte) */ 48 | 49 | int merged_len; /* contiguously merged length */ 50 | uint64_t cum_len; /* cummulatively merged length */ 51 | int last_len; /* currently saved data length */ 52 | int size; /* total ring buffer size */ 53 | 54 | /* TCP payload features */ 55 | uint32_t head_seq; 56 | uint32_t init_seq; 57 | 58 | struct fragment_ctx* fctx; 59 | }; 60 | /*----------------------------------------------------------------------------*/ 61 | uint32_t RBGetCurnum(rb_manager_t rbm); 62 | void RBPrintInfo(struct tcp_ring_buffer* buff); 63 | void RBPrintStr(struct tcp_ring_buffer* buff); 64 | void RBPrintHex(struct tcp_ring_buffer* buff); 65 | /*----------------------------------------------------------------------------*/ 66 | #ifdef ENABLELRO 67 | rb_manager_t RBManagerCreate(mtcp_manager_t mtcp, size_t chunk_size, uint32_t cnum); 68 | #else 69 | rb_manager_t RBManagerCreate(size_t chunk_size, uint32_t cnum); 70 | #endif 71 | /*----------------------------------------------------------------------------*/ 72 | struct tcp_ring_buffer* RBInit(rb_manager_t rbm, uint32_t init_seq); 73 | void RBFree(rb_manager_t rbm, struct tcp_ring_buffer* buff); 74 | uint32_t RBIsDanger(rb_manager_t rbm); 75 | /*----------------------------------------------------------------------------*/ 76 | /* data manupulation functions */ 77 | int RBPut(rb_manager_t rbm, struct tcp_ring_buffer* buff, 78 | void* data, uint32_t len , uint32_t seq); 79 | size_t RBGet(rb_manager_t rbm, struct tcp_ring_buffer* buff, size_t len); 80 | size_t RBRemove(rb_manager_t rbm, struct tcp_ring_buffer* buff, 81 | size_t len, int option); 82 | /*----------------------------------------------------------------------------*/ 83 | 84 | #endif 85 | -------------------------------------------------------------------------------- /mtcp/src/rss.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "rss.h" 10 | 11 | /*-------------------------------------------------------------*/ 12 | static void 13 | BuildKeyCache(uint32_t *cache, int cache_len) 14 | { 15 | #define NBBY 8 /* number of bits per byte */ 16 | 17 | /* Keys for system testing */ 18 | static const uint8_t key[] = { 19 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 20 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 21 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 22 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 23 | 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05 24 | }; 25 | 26 | uint32_t result = (((uint32_t)key[0]) << 24) | 27 | (((uint32_t)key[1]) << 16) | 28 | (((uint32_t)key[2]) << 8) | 29 | ((uint32_t)key[3]); 30 | 31 | uint32_t idx = 32; 32 | int i; 33 | 34 | for (i = 0; i < cache_len; i++, idx++) { 35 | uint8_t shift = (idx % NBBY); 36 | uint32_t bit; 37 | 38 | cache[i] = result; 39 | bit = ((key[idx/NBBY] << shift) & 0x80) ? 1 : 0; 40 | result = ((result << 1) | bit); 41 | } 42 | } 43 | /*-------------------------------------------------------------*/ 44 | static uint32_t 45 | GetRSSHash(in_addr_t sip, in_addr_t dip, in_port_t sp, in_port_t dp) 46 | { 47 | #define MSB32 0x80000000 48 | #define MSB16 0x8000 49 | #define KEY_CACHE_LEN 96 50 | 51 | uint32_t res = 0; 52 | int i; 53 | static int first = 1; 54 | static uint32_t key_cache[KEY_CACHE_LEN] = {0}; 55 | 56 | if (first) { 57 | BuildKeyCache(key_cache, KEY_CACHE_LEN); 58 | first = 0; 59 | } 60 | 61 | for (i = 0; i < 32; i++) { 62 | if (sip & MSB32) 63 | res ^= key_cache[i]; 64 | sip <<= 1; 65 | } 66 | for (i = 0; i < 32; i++) { 67 | if (dip & MSB32) 68 | res ^= key_cache[32+i]; 69 | dip <<= 1; 70 | } 71 | for (i = 0; i < 16; i++) { 72 | if (sp & MSB16) 73 | res ^= key_cache[64+i]; 74 | sp <<= 1; 75 | } 76 | for (i = 0; i < 16; i++) { 77 | if (dp & MSB16) 78 | res ^= key_cache[80+i]; 79 | dp <<= 1; 80 | } 81 | return res; 82 | } 83 | /*-------------------------------------------------------------------*/ 84 | /* RSS redirection table is in the little endian byte order (intel) */ 85 | /* */ 86 | /* idx: 0 1 2 3 | 4 5 6 7 | 8 9 10 11 | 12 13 14 15 | 16 17 18 19 ...*/ 87 | /* val: 3 2 1 0 | 7 6 5 4 | 11 10 9 8 | 15 14 13 12 | 19 18 17 16 ...*/ 88 | /* qid = val % num_queues */ 89 | /*-------------------------------------------------------------------*/ 90 | int 91 | GetRSSCPUCore(in_addr_t sip, in_addr_t dip, 92 | in_port_t sp, in_port_t dp, int num_queues, uint8_t endian_check) 93 | { 94 | #define RSS_BIT_MASK 0x0000007F 95 | uint32_t masked = GetRSSHash(sip, dip, sp, dp) & RSS_BIT_MASK; 96 | 97 | if (endian_check) { 98 | static const uint32_t off[4] = {3, 1, -1, -3}; 99 | masked += off[masked & 0x3]; 100 | } 101 | 102 | return (masked % num_queues); 103 | } 104 | /*-------------------------------------------------------------------*/ 105 | -------------------------------------------------------------------------------- /mtcp/src/socket.c: -------------------------------------------------------------------------------- 1 | #include "mtcp.h" 2 | #include "socket.h" 3 | #include "debug.h" 4 | 5 | /*---------------------------------------------------------------------------*/ 6 | socket_map_t 7 | AllocateSocket(mctx_t mctx, int socktype, int need_lock) 8 | { 9 | mtcp_manager_t mtcp = g_mtcp[mctx->cpu]; 10 | socket_map_t socket = NULL; 11 | 12 | #ifdef ENABLE_UCTX 13 | if (need_lock) 14 | assert(0); /* need_lock cannot be true in uctx mode */ 15 | #else 16 | if (need_lock) 17 | pthread_mutex_lock(&mtcp->ctx->smap_lock); 18 | #endif 19 | 20 | while (socket == NULL) { 21 | socket = TAILQ_FIRST(&mtcp->free_smap); 22 | if (!socket) { 23 | if (need_lock) 24 | pthread_mutex_unlock(&mtcp->ctx->smap_lock); 25 | 26 | TRACE_ERROR("The concurrent sockets are at maximum.\n"); 27 | return NULL; 28 | } 29 | 30 | TAILQ_REMOVE(&mtcp->free_smap, socket, free_smap_link); 31 | 32 | /* if there is not invalidated events, insert the socket to the end */ 33 | /* and find another socket in the free smap list */ 34 | if (socket->events) { 35 | TRACE_INFO("There are still not invalidate events remaining.\n"); 36 | TRACE_DBG("There are still not invalidate events remaining.\n"); 37 | TAILQ_INSERT_TAIL(&mtcp->free_smap, socket, free_smap_link); 38 | socket = NULL; 39 | } 40 | } 41 | #ifdef ENABLE_UCTX 42 | if (need_lock) 43 | assert(0); /* need_lock cannot be true in uctx mode */ 44 | #else 45 | if (need_lock) 46 | pthread_mutex_unlock(&mtcp->ctx->smap_lock); 47 | #endif 48 | socket->socktype = socktype; 49 | socket->opts = 0; 50 | socket->stream = NULL; 51 | socket->epoll = 0; 52 | socket->events = 0; 53 | 54 | /* 55 | * reset a few fields (needed for client socket) 56 | * addr = INADDR_ANY, port = INPORT_ANY 57 | */ 58 | memset(&socket->saddr, 0, sizeof(struct sockaddr_in)); 59 | memset(&socket->ep_data, 0, sizeof(mtcp_epoll_data_t)); 60 | 61 | return socket; 62 | } 63 | /*---------------------------------------------------------------------------*/ 64 | void 65 | FreeSocket(mctx_t mctx, int sockid, int need_lock) 66 | { 67 | mtcp_manager_t mtcp = g_mtcp[mctx->cpu]; 68 | socket_map_t socket = &mtcp->smap[sockid]; 69 | 70 | if (socket->socktype == MTCP_SOCK_UNUSED) { 71 | return; 72 | } 73 | 74 | socket->socktype = MTCP_SOCK_UNUSED; 75 | socket->epoll = MTCP_EPOLLNONE; 76 | socket->events = 0; 77 | #ifdef ENABLE_UCTX 78 | if (need_lock) 79 | assert(0); /* need_lock cannot be true in uctx mode */ 80 | #else 81 | if (need_lock) 82 | pthread_mutex_lock(&mtcp->ctx->smap_lock); 83 | #endif 84 | 85 | /* insert into free stream map */ 86 | mtcp->smap[sockid].stream = NULL; 87 | TAILQ_INSERT_TAIL(&mtcp->free_smap, socket, free_smap_link); 88 | 89 | #ifdef ENABLE_UCTX 90 | if (need_lock) 91 | assert(0); /* need_lock cannot be true in uctx mode */ 92 | #else 93 | if (need_lock) 94 | pthread_mutex_unlock(&mtcp->ctx->smap_lock); 95 | #endif 96 | } 97 | /*---------------------------------------------------------------------------*/ 98 | socket_map_t 99 | GetSocket(mctx_t mctx, int sockid) 100 | { 101 | if (sockid < 0 || sockid >= CONFIG.max_concurrency) { 102 | errno = EBADF; 103 | return NULL; 104 | } 105 | 106 | return &g_mtcp[mctx->cpu]->smap[sockid]; 107 | } 108 | -------------------------------------------------------------------------------- /apps/epproxy/src/hash.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include "config.h" 3 | /*---------------------------------------------------------------------------*/ 4 | /* (weight factor) to (# nodes in consistent hash ring) ratio 5 | * => We use the same value (16) as recommended by HAProxy for now 6 | * (https://github.com/haproxy/haproxy/blob/master/include/types/backend.h) 7 | * 8 | * The scale factor between user weight and effective weight allows smooth 9 | * weight modulation even with small weights (eg: 1). It should not be too high 10 | * though because it limits the number of servers in FWRR mode in order to 11 | * prevent any integer overflow. The max number of servers per backend is 12 | * limited to about (2^32-1)/256^2/scale ~= 65535.9999/scale. A scale of 16 13 | * looks like a good value, as it allows 4095 servers per backend while leaving 14 | * modulation steps of about 6% for servers with the lowest weight (1). 15 | */ 16 | #define CHASH_WFACTOR_SCALE 16 17 | /* max weight factor for backend servers */ 18 | #define CHASH_MAX_WFACTOR 256 19 | /* max node = SCALE * MAX_WFACTOR = 4096 -> 4 digits */ 20 | #define CHASH_NODE_MAX_DIGIT 4 21 | /*---------------------------------------------------------------------------*/ 22 | /* TODO: this should be configurable */ 23 | /* default weight factor for backend servers */ 24 | #define CHASH_DEFAULT_WFACTOR 1 25 | /*---------------------------------------------------------------------------*/ 26 | /* A full avalanche hashing function that fits well with 32-bit space 27 | * This also guarantees good distribution for small-sized numbers 28 | * (by deriving a hash value by multiplication with a large prime number) 29 | * 30 | * Provided by Bob Jonkins 31 | * (See http://burtleburtle.net/bob/hash/integer.html) 32 | */ 33 | static inline uint32_t 34 | full_aval_hash (uint32_t a) 35 | { 36 | a = (a+0x7ed55d16) + (a<<12); 37 | a = (a^0xc761c23c) ^ (a>>19); 38 | a = (a+0x165667b1) + (a<<5); 39 | a = (a+0xd3a2646c) ^ (a<<9); 40 | a = (a+0xfd7046c5) + (a<<3); 41 | a = (a^0xb55a4f09) ^ (a>>16); 42 | 43 | return a * 3221225473U; 44 | } 45 | /*---------------------------------------------------------------------------*/ 46 | /* sdbm (a public-domain reimplementation of ndbm) hash function 47 | * This is a faster version than the original one used in gawk 48 | * 49 | * (See http://www.cse.yorku.ca/~oz/hash.html) 50 | */ 51 | static inline uint32_t 52 | sdbm_hash (const char* str, uint32_t len) 53 | { 54 | uint32_t hash = 0; 55 | uint32_t i; 56 | 57 | for (i = 0; i < len; i++) 58 | hash = (str[i]) + (hash << 6) + (hash << 16) - hash; 59 | 60 | return hash; 61 | } 62 | /*---------------------------------------------------------------------------*/ 63 | typedef struct backend_node { 64 | uint32_t hash; 65 | struct backend_info* binfo; 66 | 67 | TAILQ_ENTRY (backend_node) link; 68 | 69 | } backend_node; 70 | /*---------------------------------------------------------------------------*/ 71 | /* select a backend by a given hash value */ 72 | struct backend_info* SelectNodeByHash(uint32_t hash); 73 | /* insert nodes by its hash value */ 74 | void InsertHashNodes(struct backend_info* binfo); 75 | /* remove nodes in consistent hash id by backend id */ 76 | int RemoveNodesByID(struct backend_info* binfo); 77 | /*---------------------------------------------------------------------------*/ 78 | 79 | -------------------------------------------------------------------------------- /mtcp/src/include/addr_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef __ADDR_POOL_H_ 2 | #define __ADDR_POOL_H_ 3 | 4 | #include 5 | #include 6 | 7 | #define MIN_PORT (1025) 8 | #define MAX_PORT (65535 + 1) 9 | /*----------------------------------------------------------------------------*/ 10 | struct addr_entry 11 | { 12 | struct sockaddr_in addr; 13 | TAILQ_ENTRY(addr_entry) addr_link; 14 | }; 15 | /*----------------------------------------------------------------------------*/ 16 | struct addr_map 17 | { 18 | struct addr_entry *addrmap[MAX_PORT]; 19 | }; 20 | /*----------------------------------------------------------------------------*/ 21 | struct addr_pool 22 | { 23 | struct addr_entry *pool; /* address pool */ 24 | struct addr_map *mapper; /* address map */ 25 | 26 | uint32_t addr_base; /* in host order */ 27 | int num_addr; /* number of addresses in use */ 28 | 29 | int num_entry; 30 | int num_free; 31 | int num_used; 32 | 33 | in_addr_t daddr; 34 | in_port_t dport; 35 | 36 | pthread_mutex_t lock; 37 | TAILQ_HEAD(, addr_entry) free_list; 38 | TAILQ_HEAD(, addr_entry) used_list; 39 | 40 | int num_splice; 41 | TAILQ_HEAD(, addr_entry) splice_list; 42 | 43 | TAILQ_ENTRY(addr_pool) ap_link; 44 | }; 45 | typedef struct addr_pool *addr_pool_t; 46 | /*----------------------------------------------------------------------------*/ 47 | int GetNumFree(addr_pool_t ap); 48 | int GetNumUsed(addr_pool_t ap); 49 | /*----------------------------------------------------------------------------*/ 50 | /* CreateAddressPool() */ 51 | /* Create address pool for given address range. */ 52 | /* addr_base: the base address in network order. */ 53 | /* num_addr: number of addresses to use as source IP */ 54 | /*----------------------------------------------------------------------------*/ 55 | addr_pool_t 56 | CreateAddressPool(in_addr_t addr_base, int num_addr); 57 | /*----------------------------------------------------------------------------*/ 58 | /* CreateAddressPoolPerCore() */ 59 | /* Create address pool only for the given core number. */ 60 | /* All addresses and port numbers should be in network order. */ 61 | /*----------------------------------------------------------------------------*/ 62 | addr_pool_t 63 | CreateAddressPoolPerCore(int core, int num_queues, 64 | in_addr_t saddr_base, int num_addr, in_addr_t daddr, in_port_t dport); 65 | /*----------------------------------------------------------------------------*/ 66 | void 67 | DestroyAddressPool(addr_pool_t ap); 68 | /*----------------------------------------------------------------------------*/ 69 | int 70 | FetchAddress(addr_pool_t ap, int core, int num_queues, 71 | const struct sockaddr_in *daddr, struct sockaddr_in *saddr); 72 | /*----------------------------------------------------------------------------*/ 73 | int 74 | FetchAddressPerCore(addr_pool_t ap, int core, int num_queues, 75 | const struct sockaddr_in *daddr, struct sockaddr_in *saddr); 76 | /*----------------------------------------------------------------------------*/ 77 | int 78 | FreeAddress(addr_pool_t ap, const struct sockaddr_in *addr); 79 | /*----------------------------------------------------------------------------*/ 80 | 81 | #endif /* __ADDR_POOL_H_ */ 82 | -------------------------------------------------------------------------------- /nic/setup_teardown/src/dataplane.p4: -------------------------------------------------------------------------------- 1 | /*-----------------------------------------------------------*/ 2 | primitive_action _from_wire(); 3 | primitive_action _from_host(); 4 | /*-----------------------------------------------------------*/ 5 | /* Ethernet */ 6 | #define ETHERTYPE_IPV4 0x0800 7 | #define ETHERTYPE_TEARDOWN_OFFLOAD 0x0808 8 | header_type ethernet_t { 9 | fields { 10 | dstAddr : 48; 11 | srcAddr : 48; 12 | etherType : 16; 13 | } 14 | } 15 | header ethernet_t ethernet; 16 | parser parse_ethernet { 17 | extract(ethernet); 18 | return select(latest.etherType) { 19 | ETHERTYPE_IPV4 : parse_ipv4; 20 | ETHERTYPE_TEARDOWN_OFFLOAD : parse_ipv4; 21 | default: ingress; 22 | } 23 | } 24 | /*-----------------------------------------------------------*/ 25 | /* IPv4 */ 26 | #define IP_PROT_TCP 0x06 27 | header_type ipv4_t { 28 | fields { 29 | version : 4; 30 | ihl : 4; 31 | diffserv : 8; 32 | totalLen : 16; 33 | identification : 16; 34 | flags : 3; 35 | fragOffset : 13; 36 | ttl : 8; 37 | protocol : 8; 38 | hdrChecksum : 16; 39 | srcAddr : 32; 40 | dstAddr: 32; 41 | } 42 | } 43 | header ipv4_t ipv4; 44 | parser parse_ipv4 { 45 | extract(ipv4); 46 | return select(ipv4.protocol) { 47 | IP_PROT_TCP : parse_tcp; 48 | default : ingress; 49 | } 50 | } 51 | /*-----------------------------------------------------------*/ 52 | /* TCP */ 53 | #define NET_TCP_FLAG_SYN 0x02 54 | #define ENABLE_TIMESTAMP 1 55 | header_type tcp_t { 56 | fields { 57 | sdPorts : 32; 58 | seqNo : 32; 59 | ackNo : 32; 60 | dataOffset : 4; 61 | res : 4; 62 | flags : 8; 63 | window : 16; 64 | checksum : 16; 65 | urgentPtr : 16; 66 | #if ENABLE_TIMESTAMP 67 | nop: 16; 68 | ts_op: 16; 69 | ts_val: 32; 70 | ts_ecr: 32; 71 | #endif 72 | } 73 | } 74 | header tcp_t tcp; 75 | parser parse_tcp { 76 | extract(tcp); 77 | return select(tcp.flags) { 78 | NET_TCP_FLAG_SYN : parse_tcp_opt; 79 | default : ingress; 80 | } 81 | } 82 | /*-----------------------------------------------------------*/ 83 | /* TCP extra options */ 84 | header_type tcp_opt_t { 85 | fields { 86 | wscale_op: 24; 87 | wscale_val: 8; 88 | mss_op: 16; 89 | mss_val: 16; 90 | } 91 | } 92 | header tcp_opt_t tcp_opt; 93 | parser parse_tcp_opt { 94 | extract(tcp_opt); 95 | return ingress; 96 | } 97 | /*-----------------------------------------------------------*/ 98 | action from_wire() { 99 | _from_wire(); 100 | } 101 | action from_host() { 102 | _from_host(); 103 | } 104 | /*-----------------------------------------------------------*/ 105 | table fwd_tbl { 106 | reads { 107 | standard_metadata.ingress_port : exact; 108 | } 109 | actions { 110 | from_wire; 111 | from_host; 112 | } 113 | } 114 | /*-----------------------------------------------------------*/ 115 | control ingress { 116 | apply(fwd_tbl); 117 | } 118 | /*-----------------------------------------------------------*/ 119 | parser start { 120 | return parse_ethernet; 121 | } 122 | /*-----------------------------------------------------------*/ -------------------------------------------------------------------------------- /mtcp/src/schedule.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "schedule.h" 8 | #include "lthread_api.h" 9 | #include "debug.h" 10 | 11 | #ifdef ENABLE_APP_INTERRUPT 12 | static inline int 13 | UnregisterTimerInterrupt(timer_t timerid) 14 | { 15 | int ret; 16 | struct itimerspec its = { 17 | .it_value = { 18 | .tv_sec = 0, 19 | .tv_nsec = 0, 20 | }, 21 | .it_interval = { 22 | .tv_sec = 0, 23 | .tv_nsec = 0, 24 | }, 25 | }; 26 | 27 | ret = timer_settime(timerid, 0, &its, NULL); 28 | if (ret < 0) 29 | TRACE_ERROR("Failed to unregister timer interrupt: (reason: %d)\n", errno); 30 | 31 | return ret; 32 | } 33 | 34 | static inline int 35 | RegisterTimerInterrupt(timer_t timerid, uint32_t timeout) 36 | { 37 | int ret; 38 | struct itimerspec its = { 39 | .it_value = { 40 | .tv_sec = 0, 41 | .tv_nsec = timeout, 42 | }, 43 | .it_interval = { 44 | .tv_sec = 0, 45 | .tv_nsec = 0, 46 | }, 47 | }; 48 | 49 | ret = timer_settime(timerid, 0, &its, NULL); 50 | if (ret < 0) 51 | TRACE_ERROR("Failed to register timer interrupt: (reason: %d)\n", errno); 52 | 53 | return ret; 54 | } 55 | 56 | void 57 | HandleTimerInterrupt(int sig, siginfo_t *si, void *uc) 58 | { 59 | int nif, pending_nic; 60 | 61 | if (si->si_code != SI_TIMER) { 62 | fprintf(stderr, "No\n"); 63 | } 64 | 65 | mtcp_thread_context_t ctx; 66 | 67 | ctx = (mtcp_thread_context_t) si->si_value.sival_ptr; 68 | if (ctx != NULL) { 69 | // TODO update stat schedule 70 | } 71 | 72 | /* check if there's any pending RX packet in NIC */ 73 | pending_nic = 0; 74 | for (nif = 0; nif < CONFIG.eths_num; nif++) 75 | pending_nic += ctx->mtcp_manager->iom->dev_ioctl(ctx, nif, DRV_PENDING_RX_PKT, NULL); 76 | 77 | if (!pending_nic) { 78 | RegisterTimerInterrupt(ctx->timerid, ctx->tick); 79 | 80 | return; 81 | } 82 | 83 | /* yield to stack if app is not in mTCP API */ 84 | ctx->int_app = TRUE; 85 | YieldToStack(ctx, YIELD_REASON_TIMER); 86 | } 87 | 88 | int 89 | InitTimer(mtcp_thread_context_t ctx) 90 | { 91 | int ret; 92 | struct sigevent sev; 93 | 94 | memset(&sev, -1, sizeof(struct sigevent)); 95 | sev.sigev_signo = TIMER_SIGNAL; 96 | sev.sigev_notify = SIGEV_THREAD_ID; 97 | sev.sigev_value.sival_ptr = ctx; 98 | sev._sigev_un._tid = syscall(SYS_gettid); 99 | 100 | ret = timer_create(CLOCK_REALTIME, &sev, &ctx->timerid); 101 | if (ret < 0) 102 | perror("timer_create"); 103 | 104 | /* FIXME currently using static 50us */ 105 | ctx->tick = 50000; 106 | ctx->int_app = FALSE; 107 | ctx->in_api = FALSE; 108 | 109 | UnregisterTimerInterrupt(ctx->timerid); 110 | 111 | return ret; 112 | } 113 | 114 | int 115 | DeleteTimer(mtcp_thread_context_t ctx) 116 | { 117 | int ret; 118 | 119 | ret = timer_delete(ctx->timerid); 120 | if (ret < 0) 121 | perror("timer_delete"); 122 | 123 | return ret; 124 | } 125 | 126 | #endif 127 | 128 | inline void 129 | YieldToStack(mtcp_thread_context_t ctx, int reason) 130 | { 131 | assert(mtcp); 132 | 133 | #ifdef ENABLE_APP_INTERRUPT 134 | if (reason == YIELD_REASON_TIMER) { 135 | if (ctx->int_app) 136 | UnregisterTimerInterrupt(ctx->timerid); 137 | else 138 | return; 139 | } 140 | #endif 141 | 142 | lthread_yield(); 143 | } 144 | 145 | inline void 146 | YieldToApp(mtcp_thread_context_t ctx, int interrupt) 147 | { 148 | assert(mtcp); 149 | 150 | #ifdef ENABLE_APP_INTERRUPT 151 | if (interrupt) 152 | RegisterTimerInterrupt(ctx->timerid, ctx->tick); 153 | #endif 154 | 155 | lthread_yield(); 156 | } 157 | -------------------------------------------------------------------------------- /nic/splice/src/splice.p4: -------------------------------------------------------------------------------- 1 | /*-----------------------------------------------------------*/ 2 | primitive_action lookup_state(); 3 | primitive_action apply_offload(); 4 | primitive_action fwd_from_host_vf(); 5 | /*-----------------------------------------------------------*/ 6 | /* Ethernet */ 7 | #define ETHERTYPE_IPV4 0x0800 8 | #define ETHERTYPE_ARP 0x0806 9 | #define ETHERTYPE_IPV4_OFFLOAD 0x0807 10 | header_type ethernet_t { 11 | fields { 12 | dstAddr : 48; 13 | srcAddr : 48; 14 | etherType : 16; 15 | } 16 | } 17 | header ethernet_t ethernet; 18 | parser parse_ethernet { 19 | extract(ethernet); 20 | return select(latest.etherType) { 21 | ETHERTYPE_IPV4 : parse_ipv4; 22 | ETHERTYPE_ARP : ingress; 23 | ETHERTYPE_IPV4_OFFLOAD : parse_offload; 24 | default: ingress; 25 | } 26 | } 27 | /*-----------------------------------------------------------*/ 28 | header_type offload_t { 29 | fields { 30 | frontend_srcmac : 48; 31 | frontend_nif : 16; 32 | frontend_srcip : 32; 33 | frontend_dstip : 32; 34 | frontend_sdPorts : 32; 35 | seq_offset : 32; 36 | ack_offset : 32; 37 | } 38 | } 39 | header offload_t offload; 40 | parser parse_offload { 41 | extract(offload); 42 | return parse_ipv4; 43 | } 44 | /*-----------------------------------------------------------*/ 45 | /* IPv4 */ 46 | #define IP_PROT_TCP 0x06 47 | header_type ipv4_t { 48 | fields { 49 | version : 4; 50 | ihl : 4; 51 | diffserv : 8; 52 | totalLen : 16; 53 | identification : 16; 54 | flags : 3; 55 | fragOffset : 13; 56 | ttl : 8; 57 | protocol : 8; 58 | checksum : 16; 59 | srcAddr : 32; 60 | dstAddr: 32; 61 | } 62 | } 63 | header ipv4_t ipv4; 64 | parser parse_ipv4 { 65 | extract(ipv4); 66 | return select(ipv4.protocol) { 67 | IP_PROT_TCP : parse_tcp; 68 | default : ingress; 69 | } 70 | } 71 | /*-----------------------------------------------------------*/ 72 | /* TCP */ 73 | header_type tcp_t { 74 | fields { 75 | sdPorts : 32; 76 | seqNo : 32; 77 | ackNo : 32; 78 | dataOffset : 4; 79 | res : 4; 80 | flags : 8; 81 | window : 16; 82 | checksum : 16; 83 | urgentPtr : 16; 84 | } 85 | } 86 | header tcp_t tcp; 87 | parser parse_tcp { 88 | extract(tcp); 89 | return ingress; 90 | } 91 | /*-----------------------------------------------------------*/ 92 | action fwd_to_host() { 93 | lookup_state(); 94 | } 95 | /*-----------------------------------------------------------*/ 96 | action fwd_to_network() { 97 | fwd_from_host_vf(); 98 | } 99 | /*-----------------------------------------------------------*/ 100 | action strip_and_fwd_to_network() { 101 | apply_offload(); 102 | modify_field(ethernet.etherType, ETHERTYPE_IPV4); 103 | remove_header(offload); 104 | fwd_from_host_vf(); 105 | } 106 | /*-----------------------------------------------------------*/ 107 | table fwd_tbl { 108 | reads { 109 | standard_metadata.ingress_port : exact; 110 | offload : valid; 111 | } 112 | actions { 113 | fwd_to_host; 114 | fwd_to_network; 115 | strip_and_fwd_to_network; 116 | } 117 | } 118 | /*-----------------------------------------------------------*/ 119 | control ingress { 120 | apply(fwd_tbl); 121 | } 122 | /*-----------------------------------------------------------*/ 123 | parser start { 124 | return parse_ethernet; 125 | } 126 | /*-----------------------------------------------------------*/ -------------------------------------------------------------------------------- /mtcp/src/intel_lthread/lthread_cond.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * BSD LICENSE 3 | * 4 | * Copyright(c) 2015 Intel Corporation. All rights reserved. 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in 15 | * the documentation and/or other materials provided with the 16 | * distribution. 17 | * * Neither the name of Intel Corporation nor the names of its 18 | * contributors may be used to endorse or promote products derived 19 | * from this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | */ 33 | 34 | /* 35 | * Some portions of this software may have been derived from the 36 | * https://github.com/halayli/lthread which carrys the following license. 37 | * 38 | * Copyright (C) 2012, Hasan Alayli 39 | * 40 | * Redistribution and use in source and binary forms, with or without 41 | * modification, are permitted provided that the following conditions 42 | * are met: 43 | * 1. Redistributions of source code must retain the above copyright 44 | * notice, this list of conditions and the following disclaimer. 45 | * 2. Redistributions in binary form must reproduce the above copyright 46 | * notice, this list of conditions and the following disclaimer in the 47 | * documentation and/or other materials provided with the distribution. 48 | * 49 | * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND 50 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 51 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 52 | * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 53 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 54 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 55 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 56 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 57 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 58 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 59 | * SUCH DAMAGE. 60 | */ 61 | 62 | #ifndef LTHREAD_COND_H_ 63 | #define LTHREAD_COND_H_ 64 | 65 | #include "lthread_queue.h" 66 | 67 | #define MAX_COND_NAME_SIZE 64 68 | 69 | struct lthread_cond { 70 | struct lthread_queue *blocked; 71 | struct lthread_sched *root_sched; 72 | int count; 73 | char name[MAX_COND_NAME_SIZE]; 74 | uint64_t diag_ref; /* optional ref to user diag data */ 75 | } __rte_cache_aligned; 76 | 77 | #endif /* LTHREAD_COND_H_ */ 78 | -------------------------------------------------------------------------------- /mtcp/src/Makefile.in: -------------------------------------------------------------------------------- 1 | # TODO: Make this Makefile.in pretty 2 | 3 | .PHONY: clean 4 | 5 | ### TARGET ### 6 | PS=@PSIO@ 7 | DPDK=@DPDK@ 8 | NETMAP=@NETMAP@ 9 | LRO=@LRO@ 10 | HWCSUM=@HWCSUM@ 11 | UCTX=@UCTX@ 12 | VFIO=@VFIO@ 13 | LOGGER=@LOGGER@ 14 | USE_NFP_NIC=@NFP@ 15 | APP_INTERRUPT=@APP_INTERRUPT@ 16 | MTCP_LIB_DIR=../lib 17 | MTCP_LIB=libmtcp.a 18 | MTCP_HDR_DIR=../include 19 | MTCP_HDR = mtcp_api.h mtcp_epoll.h 20 | 21 | 22 | ### GCC ### 23 | GCC=@CC@ 24 | 25 | ### FLAGS ### 26 | ifeq ($(shell uname -m),x86_64) 27 | GCC_OPT = -m64 28 | else 29 | GCC_OPT = 30 | endif 31 | GCC_OPT += -Wall -fPIC -fgnu89-inline -Werror 32 | #DBG_OPT = -DDBGMSG -DDBGFUNC -DSTREAM -DSTATE -DTSTAT -DAPP -DEPOLL 33 | #DBG_OPT = -DDBGMSG -DDBGFUNC -DSTREAM -DSTATE 34 | #DBG_OPT += -DPKTDUMP 35 | #DBG_OPT += -DDUMP_STREAM 36 | #GCC_OPT += -g -DNETSTAT -DINFO -DDBGERR -DDBGCERR 37 | GCC_OPT += -DNDEBUG -g -O3 -DNETSTAT -DINFO -DDBGERR -DDBGCERR 38 | #GCC_OPT += -DNDEBUG -g -DNETSTAT -DINFO -DDBGERR -DDBGCERR 39 | GCC_OPT += $(DBG_OPT) 40 | 41 | CFLAGS=@CFLAGS@ 42 | 43 | ### SOURCE CODE ### 44 | SRCS = core.c tcp_stream.c config.c api.c eventpoll.c socket.c pipe.c \ 45 | tcp_util.c eth_in.c ip_in.c tcp_in.c eth_out.c ip_out.c tcp_out.c \ 46 | arp.c nic_control.c timer.c cpu.c rss.c addr_pool.c fhash.c memory_mgt.c logger.c debug.c \ 47 | tcp_rb_frag_queue.c tcp_ring_buffer.c tcp_send_buffer.c tcp_sb_queue.c tcp_stream_queue.c \ 48 | psio_module.c io_module.c dpdk_module.c netmap_module.c icmp.c 49 | 50 | ifeq ($(LRO),1) 51 | GCC_OPT += -DENABLELRO 52 | endif 53 | 54 | ifeq ($(HWCSUM),0) 55 | GCC_OPT += -DDISABLE_HWCSUM 56 | endif 57 | 58 | ifeq ($(LOGGER), 1) 59 | GCC_OPT += -DENABLE_LOGGER 60 | endif 61 | 62 | ifeq ($(APP_INTERRUPT), 1) 63 | GCC_OPT += -DENABLE_APP_INTERRUPT 64 | endif 65 | 66 | ### LIBRARIES AND INCLUDES ### 67 | INC_DIR=./include 68 | INC= -I$(INC_DIR) 69 | 70 | ifeq ($(PS),1) 71 | LIBS = -lps 72 | else 73 | INC += -DDISABLE_PSIO 74 | endif 75 | 76 | ifeq ($(NETMAP),1) 77 | # do nothing 78 | else 79 | INC += -DDISABLE_NETMAP 80 | endif 81 | 82 | # PacketShader LIBRARY and HEADER 83 | PS_DIR=../../io_engine 84 | INC += -I$(PS_DIR)/include 85 | 86 | # DPDK LIBRARY and HEADER 87 | DPDK_INC=@DPDKLIBPATH@/include 88 | DPDK_LIB=@DPDKLIBPATH@/lib/ 89 | # CFLAGS for DPDK-related compilation 90 | ifeq ($(DPDK), 1) 91 | ifeq ($(UCTX), 1) 92 | GCC_OPT += -DENABLE_UCTX 93 | SRCS += intel_lthread/lthread.c intel_lthread/lthread_cond.c intel_lthread/lthread_diag.c \ 94 | intel_lthread/lthread_mutex.c intel_lthread/lthread_sched.c intel_lthread/lthread_tls.c \ 95 | intel_lthread/arch/x86/ctx.c schedule.c 96 | INC += -I./intel_lthread/ -I./intel_lthread/arch/x86/ 97 | endif 98 | ifeq ($(VFIO), 1) 99 | GCC_OPT += -DENABLE_VFIO 100 | endif 101 | 102 | ifeq ($(USE_NFP_NIC), 1) 103 | GCC_OPT += -DUSE_NFP_NIC 104 | endif 105 | 106 | DPDK_MACHINE_FLAGS = $(shell cat @DPDKLIBPATH@/include/cflags.txt) 107 | INC += ${DPDK_MACHINE_FLAGS} -I${DPDK_INC} -include $(DPDK_INC)/rte_config.h 108 | else 109 | INC += -DDISABLE_DPDK 110 | endif 111 | 112 | ifeq ($(wildcard /usr/lib/libhugetlbfs.so),) 113 | else 114 | GCC_OPT += -DHUGEPAGE 115 | endif 116 | 117 | OBJS = $(patsubst %.c,%.o,$(SRCS)) 118 | DEPS = $(patsubst %.c,.%.d,$(notdir $(SRCS))) 119 | 120 | ifeq ($V,) # no echo 121 | export MSG=@echo 122 | export HIDE=@ 123 | else 124 | export MSG=@\# 125 | export HIDE= 126 | endif 127 | 128 | ### GOALS ### 129 | all: default 130 | 131 | default: $(OBJS) $(MTCP_HDR) 132 | mkdir -p $(MTCP_LIB_DIR) 133 | ar rvs $(MTCP_LIB_DIR)/$(MTCP_LIB) $(OBJS) 134 | 135 | $(OBJS): %.o: %.c Makefile 136 | $(MSG) " CC $<" 137 | $(HIDE) $(GCC) $(CFLAGS) $(GCC_OPT) $(INC) -c $< -o $@ 138 | $(DEPS): .%.d: %.c Makefile 139 | $(HIDE) $(GCC) $(GCC_OPT) $(INC) -MM $(CFLAGS) $< > $@ 140 | 141 | -include $(DEPS) 142 | 143 | $(MTCP_HDR): 144 | cp $(INC_DIR)/$@ $(MTCP_HDR_DIR)/$@ 145 | 146 | clean: clean-library 147 | rm -f *.o *~ core 148 | rm -f .*.d 149 | 150 | clean-library: 151 | rm -f $(MTCP_LIB_DIR)/* 152 | rm -f $(MTCP_HDR_DIR)/* 153 | 154 | distclean: clean 155 | rm -f Makefile 156 | -------------------------------------------------------------------------------- /mtcp/src/tcp_sb_queue.c: -------------------------------------------------------------------------------- 1 | /* 2 | * TCP free send buffer queue - tcp_sb_queue.c/h 3 | * 4 | * EunYoung Jeong 5 | * 6 | * Part of this code borrows Click's simple queue implementation 7 | * 8 | * ============================== Click License ============================= 9 | * 10 | * Copyright (c) 1999-2000 Massachusetts Institute of Technology 11 | * 12 | * Permission is hereby granted, free of charge, to any person obtaining a 13 | * copy of this software and associated documentation files (the "Software"), 14 | * to deal in the Software without restriction, subject to the conditions 15 | * listed in the Click LICENSE file. These conditions include: you must 16 | * preserve this copyright notice, and you cannot mention the copyright 17 | * holders in advertising related to the Software without their permission. 18 | * The Software is provided WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. This 19 | * notice is a summary of the Click LICENSE file; the license in that file is 20 | * legally binding. 21 | */ 22 | 23 | #include "tcp_sb_queue.h" 24 | #include "debug.h" 25 | 26 | /*----------------------------------------------------------------------------*/ 27 | #ifndef _INDEX_TYPE_ 28 | #define _INDEX_TYPE_ 29 | typedef uint32_t index_type; 30 | typedef int32_t signed_index_type; 31 | #endif 32 | /*---------------------------------------------------------------------------*/ 33 | struct sb_queue 34 | { 35 | index_type _capacity; 36 | volatile index_type _head; 37 | volatile index_type _tail; 38 | 39 | struct tcp_send_buffer * volatile * _q; 40 | }; 41 | /*----------------------------------------------------------------------------*/ 42 | static inline index_type 43 | NextIndex(sb_queue_t sq, index_type i) 44 | { 45 | return (i != sq->_capacity ? i + 1: 0); 46 | } 47 | /*---------------------------------------------------------------------------*/ 48 | static inline index_type 49 | PrevIndex(sb_queue_t sq, index_type i) 50 | { 51 | return (i != 0 ? i - 1: sq->_capacity); 52 | } 53 | /*---------------------------------------------------------------------------*/ 54 | static inline void 55 | SBMemoryBarrier(struct tcp_send_buffer * volatile buf, volatile index_type index) 56 | { 57 | __asm__ volatile("" : : "m" (buf), "m" (index)); 58 | } 59 | /*---------------------------------------------------------------------------*/ 60 | sb_queue_t 61 | CreateSBQueue(int capacity) 62 | { 63 | sb_queue_t sq; 64 | 65 | sq = (sb_queue_t)calloc(1, sizeof(struct sb_queue)); 66 | if (!sq) 67 | return NULL; 68 | 69 | sq->_q = (struct tcp_send_buffer **) 70 | calloc(capacity + 1, sizeof(struct tcp_send_buffer *)); 71 | if (!sq->_q) { 72 | free(sq); 73 | return NULL; 74 | } 75 | 76 | sq->_capacity = capacity; 77 | sq->_head = sq->_tail = 0; 78 | 79 | return sq; 80 | } 81 | /*---------------------------------------------------------------------------*/ 82 | void 83 | DestroySBQueue(sb_queue_t sq) 84 | { 85 | if (!sq) 86 | return; 87 | 88 | if (sq->_q) { 89 | free((void *)sq->_q); 90 | sq->_q = NULL; 91 | } 92 | 93 | free(sq); 94 | } 95 | /*---------------------------------------------------------------------------*/ 96 | int 97 | SBEnqueue(sb_queue_t sq, struct tcp_send_buffer *buf) 98 | { 99 | index_type h = sq->_head; 100 | index_type t = sq->_tail; 101 | index_type nt = NextIndex(sq, t); 102 | 103 | if (nt != h) { 104 | sq->_q[t] = buf; 105 | SBMemoryBarrier(sq->_q[t], sq->_tail); 106 | sq->_tail = nt; 107 | return 0; 108 | } 109 | 110 | TRACE_ERROR("Exceed capacity of buf queue!\n"); 111 | return -1; 112 | } 113 | /*---------------------------------------------------------------------------*/ 114 | struct tcp_send_buffer * 115 | SBDequeue(sb_queue_t sq) 116 | { 117 | index_type h = sq->_head; 118 | index_type t = sq->_tail; 119 | 120 | if (h != t) { 121 | struct tcp_send_buffer *buf = sq->_q[h]; 122 | SBMemoryBarrier(sq->_q[h], sq->_head); 123 | sq->_head = NextIndex(sq, h); 124 | assert(buf); 125 | 126 | return buf; 127 | } 128 | 129 | return NULL; 130 | } 131 | /*---------------------------------------------------------------------------*/ 132 | -------------------------------------------------------------------------------- /apps/epproxy/src/epproxy.h: -------------------------------------------------------------------------------- 1 | #ifndef __EPPROXY_H_ 2 | #define __EPPROXY_H_ 3 | #define _LARGEFILE64_SOURCE 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include "http_stream.h" 17 | #include "persist.h" 18 | #include "backend_pool.h" 19 | /*----------------------------------------------------------------------------*/ 20 | #define USE_MTCP 1 21 | #if USE_MTCP 22 | #include 23 | #else 24 | #include 25 | #include 26 | #define MAX_CONCURRENCY_LIMIT 1000000 27 | #endif 28 | /*---------------------------------------------------------------------------*/ 29 | /* configurable parameters for epproxy */ 30 | #define MAX_PROXY_NUM 1 31 | #define MAX_BACKEND_NUM 1024 32 | #define MAX_BACKEND_POOL_NUM 4 33 | #define MAX_COOKIE_NAME_LEN 20 34 | #define MAX_SERVER_NAME_LEN 10 35 | #define PB_ALLOCSIZE 8192 36 | /*---------------------------------------------------------------------------*/ 37 | // #define MAX_IP_LEN 15 38 | #define MAX_IPADDR_STRLEN 15 39 | #define MAX_PERSIST_METHOD_LEN 14 40 | /*----------------------------------------------------------------------------*/ 41 | #define COOKIE_HEADER_LEN 13 /* = 12 for "Set-Cookie: " + 1 for "=" */ 42 | #define COOKIE_PREFIX_LEN COOKIE_HEADER_LEN + MAX_COOKIE_NAME_LEN 43 | #define COOKIE_TOTAL_LEN COOKIE_PREFIX_LEN + MAX_SERVER_NAME_LEN 44 | /*----------------------------------------------------------------------------*/ 45 | #define PB_READSIZE PB_ALLOCSIZE - MAX_IPADDR_STRLEN - COOKIE_TOTAL_LEN 46 | /*----------------------------------------------------------------------------*/ 47 | extern const char* const config_opts[]; 48 | extern const char* const persist_opts[]; 49 | /*---------------------------------------------------------------------------*/ 50 | enum {BL_SINGLE, BL_ROUNDROBIN, BL_LEAST, BL_URI}; 51 | /*---------------------------------------------------------------------------*/ 52 | typedef struct thread_context 53 | { 54 | #if USE_MTCP 55 | mctx_t mctx; /* mtcp context */ 56 | int cpu; /* CPU core number */ 57 | #endif 58 | int listener; /* listener socket */ 59 | int ep; /* epoll socket */ 60 | int rr_count; /* round-robin counter */ 61 | 62 | http_stream *stream; /* per-socket HTTP stream structure */ 63 | 64 | http_buf *hbmap; /* per-stream HTTP buffer structure */ 65 | TAILQ_HEAD (, http_buf) free_hbmap; /* list of free HTTP buffers */ 66 | 67 | struct sticky_table sticky_map; /* sticky table */ 68 | 69 | } thread_context; 70 | /*----------------------------------------------------------------------------*/ 71 | struct proxy_context { 72 | /* listening address of the frontend server */ 73 | struct sockaddr_in listen_addr; 74 | 75 | /* load balancing method */ 76 | int balance; 77 | 78 | /* backend server pool information */ 79 | struct backend_pool bpool[MAX_BACKEND_POOL_NUM]; 80 | /* consistent hash node map for backend servers */ 81 | TAILQ_HEAD (, backend_node) bnode_hmap; 82 | 83 | /* number of persistent backend connections */ 84 | int conn_per_backend; 85 | 86 | /* session persistence method */ 87 | int persist_method; 88 | /* session persistence cookie name */ 89 | char persist_cookie[MAX_COOKIE_NAME_LEN + 1]; 90 | 91 | /* config parameter */ 92 | uint8_t listen_done:1, 93 | balance_done:1, 94 | // backend_done:1, 95 | conn_pool_done:1, 96 | persist_done:1; 97 | 98 | int backend_num; 99 | 100 | }; 101 | /*---------------------------------------------------------------------------*/ 102 | struct config { 103 | int proxy_num; 104 | struct proxy_context pconf[MAX_PROXY_NUM]; 105 | } g_conf; 106 | /*---------------------------------------------------------------------------*/ 107 | struct proxy_context *g_prx_ctx; 108 | /*---------------------------------------------------------------------------*/ 109 | #endif 110 | -------------------------------------------------------------------------------- /mtcp/src/intel_lthread/arch/x86/ctx.c: -------------------------------------------------------------------------------- 1 | /*- 2 | * BSD LICENSE 3 | * 4 | * Copyright(c) 2015 Intel Corporation. All rights reserved. 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in 15 | * the documentation and/or other materials provided with the 16 | * distribution. 17 | * * Neither the name of Intel Corporation nor the names of its 18 | * contributors may be used to endorse or promote products derived 19 | * from this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | */ 33 | 34 | /* 35 | * https://github.com/halayli/lthread which carries the following license. 36 | * 37 | * Copyright (C) 2012, Hasan Alayli 38 | * 39 | * Redistribution and use in source and binary forms, with or without 40 | * modification, are permitted provided that the following conditions 41 | * are met: 42 | * 1. Redistributions of source code must retain the above copyright 43 | * notice, this list of conditions and the following disclaimer. 44 | * 2. Redistributions in binary form must reproduce the above copyright 45 | * notice, this list of conditions and the following disclaimer in the 46 | * documentation and/or other materials provided with the distribution. 47 | * 48 | * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND 49 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 50 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 51 | * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 52 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 53 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 54 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 55 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 56 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 57 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 58 | * SUCH DAMAGE. 59 | */ 60 | 61 | 62 | 63 | #if defined(__x86_64__) 64 | __asm__ ( 65 | ".text\n" 66 | ".p2align 4,,15\n" 67 | ".globl ctx_switch\n" 68 | ".globl _ctx_switch\n" 69 | "ctx_switch:\n" 70 | "_ctx_switch:\n" 71 | " movq %rsp, 0(%rsi) # save stack_pointer\n" 72 | " movq %rbp, 8(%rsi) # save frame_pointer\n" 73 | " movq (%rsp), %rax # save insn_pointer\n" 74 | " movq %rax, 16(%rsi)\n" 75 | " movq %rbx, 24(%rsi)\n # save rbx,r12-r15\n" 76 | " movq 24(%rdi), %rbx\n" 77 | " movq %r15, 56(%rsi)\n" 78 | " movq %r14, 48(%rsi)\n" 79 | " movq 48(%rdi), %r14\n" 80 | " movq 56(%rdi), %r15\n" 81 | " movq %r13, 40(%rsi)\n" 82 | " movq %r12, 32(%rsi)\n" 83 | " movq 32(%rdi), %r12\n" 84 | " movq 40(%rdi), %r13\n" 85 | " movq 0(%rdi), %rsp # restore stack_pointer\n" 86 | " movq 16(%rdi), %rax # restore insn_pointer\n" 87 | " movq 8(%rdi), %rbp # restore frame_pointer\n" 88 | " movq %rax, (%rsp)\n" 89 | " ret\n" 90 | ); 91 | #else 92 | #pragma GCC error "__x86_64__ is not defined" 93 | #endif 94 | -------------------------------------------------------------------------------- /nic/setup_teardown/src/include/common.h: -------------------------------------------------------------------------------- 1 | #ifndef COMMON_H 2 | #define COMMON_H 3 | /*------------------------------------------------------------------------*/ 4 | #include 5 | #include 6 | #include 7 | #include 8 | /*------------------------------------------------------------------------*/ 9 | /* user-configurable parameters */ 10 | /*------------------------------------------------------------------------*/ 11 | /* enable connection setup offload */ 12 | #define ENABLE_SETUP_OFFLOAD 1 13 | /* enable connetion teardown offload */ 14 | #define ENABLE_TEARDOWN_OFFLOAD 1 15 | /* (teardown offload) enable retransmission of offloaded payload */ 16 | #define ENABLE_TEARDOWN_RTO 1 17 | #if ENABLE_TEARDOWN_RTO 18 | #define DEFAULT_TS_RTO TS_500MS 19 | #define DEFAULT_RTO_EPOCH (DEFAULT_TS_RTO >> BITSHIFT_RTO_GROUPS) 20 | #endif 21 | /*------------------------------------------------------------------------*/ 22 | /* enable TCP timestamp option */ 23 | #define ENABLE_TIMESTAMP 1 24 | /*------------------------------------------------------------------------*/ 25 | /* enable timewait in connection teardown */ 26 | #define ENABLE_TEARDOWN_TIMEWAIT 0 27 | #if ENABLE_TEARDOWN_TIMEWAIT 28 | /* TIME_WAIT duration = (DEFAULT_TS_RTO * TW_TIMER_CYCLE) */ 29 | #define TW_TIMER_CYCLE 3 30 | #define INIT_TWCYCLE (TW_TIMER_CYCLE << 28) 31 | #define TWCYCLE_1 (1 << 28) 32 | #endif 33 | /*------------------------------------------------------------------------*/ 34 | /* enable early retransmission on payload retransmission from client */ 35 | #define EARLY_REXMIT 0 36 | /*------------------------------------------------------------------------*/ 37 | #define FILTER_BY_ADDR 1 38 | /*------------------------------------------------------------------------*/ 39 | /* TCP option values (FIXME: hard-coded default values of mTCP) */ 40 | #define TCP_WSCALE_DEFAULT 7 41 | #define TCP_MSS_DEFAULT 1460 42 | /*------------------------------------------------------------------------*/ 43 | /* implementation-specific parameters */ 44 | /*------------------------------------------------------------------------*/ 45 | #define NUM_RTO_GROUPS 8 /* should be power of 2 */ 46 | #define BITMASK_RTO_GROUPS (NUM_RTO_GROUPS - 1) 47 | #define BITSHIFT_RTO_GROUPS 3 /* 2^3 = 8 */ 48 | /*------------------------------------------------------------------------*/ 49 | #define NUM_HOSTS 4 50 | #define NUM_PORTS (1 << 16) 51 | #define MAX_FLOWS NUM_HOSTS * NUM_PORTS 52 | /*------------------------------------------------------------------------*/ 53 | #define MAX_PKTNUM 1 54 | #define PKTBUF_SIZE ((DEFAULT_MTU + sizeof(uint32_t) - 1) / sizeof(uint32_t)) 55 | /*------------------------------------------------------------------------*/ 56 | /* constant/type definitions */ 57 | /*------------------------------------------------------------------------*/ 58 | #define TRUE 1 59 | #define FALSE 0 60 | #define NULL 0 61 | /*------------------------------------------------------------------------*/ 62 | #define NET_TCP_FLAG_SYN 0x02 63 | #define NET_TCP_FLAG_ACK 0x10 64 | #define NET_TCP_FLAG_SYNACK 0x12 65 | #define NET_TCP_FLAG_FIN 0x01 66 | #define NET_TCP_FLAG_RST 0x04 67 | /*------------------------------------------------------------------------*/ 68 | #define ETHERTYPE_IPV4 0x0800 69 | #define ETHERTYPE_TEARDOWN_OFFLOAD 0x0808 70 | #define ETHERTYPE_SETUP_OFFLOAD 0x0809 71 | #define ETHERTYPE_SETUP_OFFLOAD_CTRL 0x080B 72 | #define ETHERTYPE_OFFLOAD_INIT_CTRL 0x080C 73 | /*------------------------------------------------------------------------*/ 74 | #define DEFAULT_MTU 1514 75 | #define DEFAULT_MSS 1460 76 | #define DEFAULT_HDR 54 77 | #define ETH_HDRLEN 14 78 | #define TCP_HDRLEN 20 79 | #define TCPIP_HDRLEN 40 80 | /*------------------------------------------------------------------------*/ 81 | #define ISN_FILTER 0x00FFFFFF 82 | /*------------------------------------------------------------------------*/ 83 | 84 | #define MIN(a,b) ((a < b)? a : b) 85 | #define SWAP(ports) ((ports >> 16) | ((ports & 0xFFFF) << 16)) 86 | /*------------------------------------------------------------------------*/ 87 | #endif 88 | -------------------------------------------------------------------------------- /mtcp/src/tcp_rb_frag_queue.c: -------------------------------------------------------------------------------- 1 | /* 2 | * TCP free fragment queue for ring buffer - tcp_rb_frag_queue.c/h 3 | * 4 | * EunYoung Jeong 5 | * 6 | * Part of this code borrows Click's simple queue implementation 7 | * 8 | * ============================== Click License ============================= 9 | * 10 | * Copyright (c) 1999-2000 Massachusetts Institute of Technology 11 | * 12 | * Permission is hereby granted, free of charge, to any person obtaining a 13 | * copy of this software and associated documentation files (the "Software"), 14 | * to deal in the Software without restriction, subject to the conditions 15 | * listed in the Click LICENSE file. These conditions include: you must 16 | * preserve this copyright notice, and you cannot mention the copyright 17 | * holders in advertising related to the Software without their permission. 18 | * The Software is provided WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. This 19 | * notice is a summary of the Click LICENSE file; the license in that file is 20 | * legally binding. 21 | */ 22 | 23 | #include "tcp_rb_frag_queue.h" 24 | #include "debug.h" 25 | 26 | /*----------------------------------------------------------------------------*/ 27 | #ifndef _INDEX_TYPE_ 28 | #define _INDEX_TYPE_ 29 | typedef uint32_t index_type; 30 | typedef int32_t signed_index_type; 31 | #endif 32 | /*---------------------------------------------------------------------------*/ 33 | struct rb_frag_queue 34 | { 35 | index_type _capacity; 36 | volatile index_type _head; 37 | volatile index_type _tail; 38 | 39 | struct fragment_ctx * volatile * _q; 40 | }; 41 | /*----------------------------------------------------------------------------*/ 42 | static inline index_type 43 | NextIndex(rb_frag_queue_t rb_fragq, index_type i) 44 | { 45 | return (i != rb_fragq->_capacity ? i + 1: 0); 46 | } 47 | /*---------------------------------------------------------------------------*/ 48 | static inline index_type 49 | PrevIndex(rb_frag_queue_t rb_fragq, index_type i) 50 | { 51 | return (i != 0 ? i - 1: rb_fragq->_capacity); 52 | } 53 | /*---------------------------------------------------------------------------*/ 54 | static inline void 55 | RBFragMemoryBarrier(struct fragment_ctx * volatile frag, volatile index_type index) 56 | { 57 | __asm__ volatile("" : : "m" (frag), "m" (index)); 58 | } 59 | /*---------------------------------------------------------------------------*/ 60 | rb_frag_queue_t 61 | CreateRBFragQueue(int capacity) 62 | { 63 | rb_frag_queue_t rb_fragq; 64 | 65 | rb_fragq = (rb_frag_queue_t)calloc(1, sizeof(struct rb_frag_queue)); 66 | if (!rb_fragq) 67 | return NULL; 68 | 69 | rb_fragq->_q = (struct fragment_ctx **) 70 | calloc(capacity + 1, sizeof(struct fragment_ctx *)); 71 | if (!rb_fragq->_q) { 72 | free(rb_fragq); 73 | return NULL; 74 | } 75 | 76 | rb_fragq->_capacity = capacity; 77 | rb_fragq->_head = rb_fragq->_tail = 0; 78 | 79 | return rb_fragq; 80 | } 81 | /*---------------------------------------------------------------------------*/ 82 | void 83 | DestroyRBFragQueue(rb_frag_queue_t rb_fragq) 84 | { 85 | if (!rb_fragq) 86 | return; 87 | 88 | if (rb_fragq->_q) { 89 | free((void *)rb_fragq->_q); 90 | rb_fragq->_q = NULL; 91 | } 92 | 93 | free(rb_fragq); 94 | } 95 | /*---------------------------------------------------------------------------*/ 96 | int 97 | RBFragEnqueue(rb_frag_queue_t rb_fragq, struct fragment_ctx *frag) 98 | { 99 | index_type h = rb_fragq->_head; 100 | index_type t = rb_fragq->_tail; 101 | index_type nt = NextIndex(rb_fragq, t); 102 | 103 | if (nt != h) { 104 | rb_fragq->_q[t] = frag; 105 | RBFragMemoryBarrier(rb_fragq->_q[t], rb_fragq->_tail); 106 | rb_fragq->_tail = nt; 107 | return 0; 108 | } 109 | 110 | TRACE_ERROR("Exceed capacity of frag queue!\n"); 111 | return -1; 112 | } 113 | /*---------------------------------------------------------------------------*/ 114 | struct fragment_ctx * 115 | RBFragDequeue(rb_frag_queue_t rb_fragq) 116 | { 117 | index_type h = rb_fragq->_head; 118 | index_type t = rb_fragq->_tail; 119 | 120 | if (h != t) { 121 | struct fragment_ctx *frag = rb_fragq->_q[h]; 122 | RBFragMemoryBarrier(rb_fragq->_q[h], rb_fragq->_head); 123 | rb_fragq->_head = NextIndex(rb_fragq, h); 124 | assert(frag); 125 | 126 | return frag; 127 | } 128 | 129 | return NULL; 130 | } 131 | /*---------------------------------------------------------------------------*/ 132 | -------------------------------------------------------------------------------- /nic/splice/src/include/packet_management.h: -------------------------------------------------------------------------------- 1 | #ifndef PACKET_MANAGEMENT_H 2 | #define PACKET_MANAGEMENT_H 3 | 4 | /*------------------------------------------------------------------------*/ 5 | /* Ethernet Packet Management */ 6 | /*------------------------------------------------------------------------*/ 7 | struct packet_tx_control { 8 | union { 9 | __packed struct { 10 | uint16_t close_reason; 11 | uint32_t ip_addr; 12 | uint16_t port; 13 | uint32_t unused; 14 | uint16_t eth_type; 15 | /* For RSS ap search */ 16 | uint32_t dst_addr; 17 | uint16_t dst_port; 18 | }; 19 | uint32_t __raw[5]; 20 | }; 21 | }; 22 | /*------------------------------------------------------------------------*/ 23 | static void 24 | send_control_packet(uint32_t out_port, 25 | uint16_t close_reason, 26 | uint32_t ip_addr, uint16_t port, 27 | uint32_t dst_addr, uint16_t dst_port) 28 | { 29 | int i; 30 | /* We write to packet data here and copy it into the ctm buffer */ 31 | __lmem struct packet_tx_control Pdata; 32 | 33 | /* Build up the packet */ 34 | reg_zero(Pdata.__raw, sizeof(struct packet_tx_control)); 35 | 36 | Pdata.close_reason = close_reason; 37 | Pdata.ip_addr = ip_addr; 38 | Pdata.port = port; 39 | Pdata.unused = 0x0000; 40 | Pdata.eth_type = CTRL_P_SPLICE_FINISH; 41 | Pdata.dst_addr = dst_addr; 42 | Pdata.dst_port = dst_port; 43 | 44 | /* TO_HOST */ 45 | if ((out_port & BASE_VF0_PORT_ID) == BASE_VF0_PORT_ID) { 46 | send_packet_host_anyway(&(Pdata.__raw), 47 | (out_port - BASE_VF0_PORT_ID), 48 | sizeof(struct packet_tx_control)); 49 | } 50 | /* Wrong Port (We send control packets only to the host) */ 51 | else { 52 | /* Given port is not VF */ 53 | local_csr_write(local_csr_mailbox_2, 0xFFFF0000); 54 | } 55 | } 56 | 57 | /*------------------------------------------------------------------------*/ 58 | /* TCP Packet Management */ 59 | /*------------------------------------------------------------------------*/ 60 | #define TCP_DATA_LEN 1 61 | struct packet_tx_tcp { 62 | union { 63 | __packed struct { 64 | uint16_t eth_dst_hi; 65 | uint32_t eth_dst_lo; 66 | uint32_t eth_src_hi; 67 | uint16_t eth_src_lo; 68 | uint16_t eth_type; 69 | struct ip4_hdr ip; 70 | struct tcp_hdr tcp; 71 | uint8_t tcp_data[TCP_DATA_LEN]; 72 | }; 73 | uint32_t __raw[16]; 74 | }; 75 | }; 76 | /*------------------------------------------------------------------------*/ 77 | static void 78 | send_tcp_packet(uint32_t out_port, 79 | uint16_t dst_mac_hi, uint32_t dst_mac_lo, 80 | uint32_t src_mac_hi, uint16_t src_mac_lo, 81 | uint32_t srcAddr, uint32_t dstAddr, 82 | #if UPDATE_CHECKSUM 83 | uint16_t ip_sum, uint32_t tcp_sum, 84 | #endif 85 | uint16_t tcp_flag, 86 | uint16_t srcPort, uint16_t dstPort, 87 | uint32_t seqNum, uint32_t ackNum) 88 | { 89 | int i; 90 | /* We write to packet data here and copy it into the ctm buffer */ 91 | __lmem struct packet_tx_tcp Pdata; 92 | 93 | /* Build up the packet */ 94 | reg_zero(Pdata.__raw, sizeof(struct packet_tx_tcp)); 95 | 96 | Pdata.eth_dst_hi = dst_mac_hi; 97 | Pdata.eth_dst_lo = dst_mac_lo; 98 | Pdata.eth_src_hi = src_mac_hi; 99 | Pdata.eth_src_lo = src_mac_lo; 100 | Pdata.eth_type = NET_ETH_TYPE_IPV4; 101 | 102 | Pdata.ip.ver = 4; 103 | Pdata.ip.hl = 5; 104 | Pdata.ip.tos = 0; 105 | Pdata.ip.len = sizeof(Pdata.ip) + 106 | sizeof(Pdata.tcp) + sizeof(Pdata.tcp_data); 107 | Pdata.ip.frag = 0; 108 | Pdata.ip.ttl = 64; 109 | Pdata.ip.proto = 0x06; 110 | Pdata.ip.src = srcAddr; 111 | Pdata.ip.dst = dstAddr; 112 | 113 | Pdata.tcp.sport = srcPort; 114 | Pdata.tcp.dport = dstPort; 115 | Pdata.tcp.seq = seqNum; 116 | Pdata.tcp.ack = ackNum; 117 | Pdata.tcp.off = 5; 118 | Pdata.tcp.flags = tcp_flag; 119 | Pdata.tcp.win = 6000; 120 | Pdata.tcp.urp = 0; 121 | 122 | #if UPDATE_CHECKSUM 123 | Pdata.ip.sum = ip_sum; 124 | Pdata.tcp.sum = tcp_sum; 125 | #endif 126 | 127 | /* TO_WIRE */ 128 | if ((out_port & BASE_VF0_PORT_ID) == BASE_PHY0_PORT_ID) { 129 | send_packet_wire(&(Pdata.__raw), 130 | (out_port ? 131 | 1 : 0), 132 | sizeof(struct packet_tx_tcp)); 133 | } 134 | /* Wrong Port */ 135 | else { 136 | /* Given port is neither PF nor VF */ 137 | local_csr_write(local_csr_mailbox_2, 0xFFFF0000); 138 | } 139 | } 140 | /*------------------------------------------------------------------------*/ 141 | 142 | #endif /* PACKET_MANAGEMENT_H */ 143 | -------------------------------------------------------------------------------- /mtcp/src/include/tcp_in.h: -------------------------------------------------------------------------------- 1 | #ifndef __TCP_IN_H_ 2 | #define __TCP_IN_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "mtcp.h" 10 | #include "fhash.h" 11 | 12 | #define TCP_FLAG_FIN 0x01 // 0000 0001 13 | #define TCP_FLAG_SYN 0x02 // 0000 0010 14 | #define TCP_FLAG_RST 0x04 // 0000 0100 15 | #define TCP_FLAG_PSH 0x08 // 0000 1000 16 | #define TCP_FLAG_ACK 0x10 // 0001 0000 17 | #define TCP_FLAG_URG 0x20 // 0010 0000 18 | #define TCP_FLAG_SACK 0x40 // 0100 0000 19 | #define TCP_FLAG_WACK 0x80 // 1000 0000 20 | 21 | #define TCP_OPT_FLAG_MSS 0x02 // 0000 0010 22 | #define TCP_OPT_FLAG_WSCALE 0x04 // 0000 0100 23 | #define TCP_OPT_FLAG_SACK_PERMIT 0x08 // 0000 1000 24 | #define TCP_OPT_FLAG_SACK 0x10 // 0001 0000 25 | #define TCP_OPT_FLAG_TIMESTAMP 0x20 // 0010 0000 26 | 27 | #define TCP_OPT_MSS_LEN 4 28 | #define TCP_OPT_WSCALE_LEN 3 29 | #define TCP_OPT_SACK_PERMIT_LEN 2 30 | #define TCP_OPT_SACK_LEN 10 31 | #define TCP_OPT_TIMESTAMP_LEN 10 32 | 33 | #define TCP_DEFAULT_MSS 1460 34 | #define TCP_DEFAULT_WSCALE 7 35 | #define TCP_INITIAL_WINDOW 14600 // initial window size 36 | 37 | #define TCP_SEQ_LT(a,b) ((int32_t)((a)-(b)) < 0) 38 | #define TCP_SEQ_LEQ(a,b) ((int32_t)((a)-(b)) <= 0) 39 | #define TCP_SEQ_GT(a,b) ((int32_t)((a)-(b)) > 0) 40 | #define TCP_SEQ_GEQ(a,b) ((int32_t)((a)-(b)) >= 0) 41 | #define TCP_SEQ_BETWEEN(a,b,c) (TCP_SEQ_GEQ(a,b) && TCP_SEQ_LEQ(a,c)) 42 | 43 | /* convert timeval to timestamp (precision: 1 ms) */ 44 | #define HZ 1000 45 | #define TIME_TICK (1000000/HZ) // in us 46 | #define TIMEVAL_TO_TS(t) (uint32_t)((t)->tv_sec * HZ + \ 47 | ((t)->tv_usec / TIME_TICK)) 48 | 49 | #define TS_TO_USEC(t) ((t) * TIME_TICK) 50 | #define TS_TO_MSEC(t) (TS_TO_USEC(t) / 1000) 51 | 52 | #define USEC_TO_TS(t) ((t) / TIME_TICK) 53 | #define MSEC_TO_TS(t) (USEC_TO_TS((t) * 1000)) 54 | #define SEC_TO_TS(t) (t * HZ) 55 | 56 | #define SEC_TO_USEC(t) ((t) * 1000000) 57 | #define SEC_TO_MSEC(t) ((t) * 1000) 58 | #define MSEC_TO_USEC(t) ((t) * 1000) 59 | #define USEC_TO_SEC(t) ((t) / 1000000) 60 | //#define TCP_TIMEWAIT (MSEC_TO_USEC(5000) / TIME_TICK) // 5s 61 | #define TCP_TIMEWAIT 0 62 | #define TCP_INITIAL_RTO (MSEC_TO_USEC(500) / TIME_TICK) // 500ms 63 | #define TCP_FIN_RTO (MSEC_TO_USEC(500) / TIME_TICK) // 500ms 64 | #define TCP_TIMEOUT (MSEC_TO_USEC(30000) / TIME_TICK) // 30s 65 | 66 | #define TCP_MAX_RTX 16 67 | #define TCP_MAX_SYN_RETRY 7 68 | #define TCP_MAX_BACKOFF 7 69 | 70 | enum tcp_state 71 | { 72 | TCP_ST_CLOSED = 0, 73 | TCP_ST_LISTEN = 1, 74 | TCP_ST_SYN_SENT = 2, 75 | TCP_ST_SYN_RCVD = 3, 76 | TCP_ST_ESTABLISHED = 4, 77 | TCP_ST_FIN_WAIT_1 = 5, 78 | TCP_ST_FIN_WAIT_2 = 6, 79 | TCP_ST_CLOSE_WAIT = 7, 80 | TCP_ST_CLOSING = 8, 81 | TCP_ST_LAST_ACK = 9, 82 | TCP_ST_TIME_WAIT = 10 83 | }; 84 | 85 | enum tcp_option 86 | { 87 | TCP_OPT_END = 0, 88 | TCP_OPT_NOP = 1, 89 | TCP_OPT_MSS = 2, 90 | TCP_OPT_WSCALE = 3, 91 | TCP_OPT_SACK_PERMIT = 4, 92 | TCP_OPT_SACK = 5, 93 | TCP_OPT_TIMESTAMP = 8 94 | }; 95 | 96 | enum tcp_close_reason 97 | { 98 | TCP_NOT_CLOSED = 0, 99 | TCP_ACTIVE_CLOSE = 1, 100 | TCP_PASSIVE_CLOSE = 2, 101 | TCP_CONN_FAIL = 3, 102 | TCP_CONN_LOST = 4, 103 | TCP_RESET = 5, 104 | TCP_NO_MEM = 6, 105 | TCP_NOT_ACCEPTED = 7, 106 | TCP_TIMEDOUT = 8 107 | }; 108 | 109 | void 110 | ParseTCPOptions(tcp_stream *cur_stream, 111 | uint32_t cur_ts, uint8_t *tcpopt, int len); 112 | 113 | extern inline int 114 | ProcessTCPUplink(mtcp_manager_t mtcp, uint32_t cur_ts, tcp_stream *cur_stream, 115 | const struct tcphdr *tcph, uint32_t seq, uint32_t ack_seq, 116 | uint8_t *payload, int payloadlen, uint32_t window); 117 | 118 | int 119 | ProcessCUSTOMPacket(struct mtcp_manager *mtcp, uint32_t cur_ts, const int ifidx, 120 | const struct iphdr* iph, int ip_len); 121 | int 122 | ProcessTCPPacket(struct mtcp_manager *mtcp, uint32_t cur_ts, const int ifidx, 123 | const struct iphdr* iph, int ip_len); 124 | uint16_t 125 | TCPCalcChecksum(uint16_t *buf, uint16_t len, uint32_t saddr, uint32_t daddr); 126 | 127 | #if OPPORTUNISTIC_STREAM_ALLOC 128 | inline int 129 | CopyFromUser(mtcp_manager_t mtcp, tcp_stream *cur_stream, const char *buf, int len); 130 | inline int 131 | ProcessTCPPayload(mtcp_manager_t mtcp, tcp_stream *cur_stream, 132 | uint32_t cur_ts, uint8_t *payload, uint32_t seq, int payloadlen); 133 | #endif 134 | 135 | #endif /* __TCP_IN_H_ */ 136 | -------------------------------------------------------------------------------- /mtcp/src/intel_lthread/lthread.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * BSD LICENSE 3 | * 4 | * Copyright(c) 2015 Intel Corporation. All rights reserved. 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in 15 | * the documentation and/or other materials provided with the 16 | * distribution. 17 | * * Neither the name of Intel Corporation nor the names of its 18 | * contributors may be used to endorse or promote products derived 19 | * from this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | */ 33 | 34 | /* 35 | * Some portions of this software is derived from the 36 | * https://github.com/halayli/lthread which carrys the following license. 37 | * 38 | * Copyright (C) 2012, Hasan Alayli 39 | * 40 | * Redistribution and use in source and binary forms, with or without 41 | * modification, are permitted provided that the following conditions 42 | * are met: 43 | * 1. Redistributions of source code must retain the above copyright 44 | * notice, this list of conditions and the following disclaimer. 45 | * 2. Redistributions in binary form must reproduce the above copyright 46 | * notice, this list of conditions and the following disclaimer in the 47 | * documentation and/or other materials provided with the distribution. 48 | * 49 | * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND 50 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 51 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 52 | * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 53 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 54 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 55 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 56 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 57 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 58 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 59 | * SUCH DAMAGE. 60 | */ 61 | 62 | #ifndef LTHREAD_H_ 63 | #define LTHREAD_H_ 64 | 65 | #include 66 | 67 | #include "lthread_api.h" 68 | #include "lthread_diag.h" 69 | 70 | typedef struct lthread * lthread_t; 71 | struct lthread; 72 | struct lthread_sched; 73 | 74 | /* function to be called when a context function returns */ 75 | typedef void (*lthread_exit_func) (struct lthread *); 76 | 77 | void _lthread_exit_handler(struct lthread *lt); 78 | 79 | void lthread_set_funcname(const char *f); 80 | 81 | void _lthread_sched_busy_sleep(struct lthread *lt, uint64_t nsecs); 82 | 83 | int _lthread_desched_sleep(struct lthread *lt); 84 | 85 | void _lthread_free(struct lthread *lt); 86 | 87 | struct lthread_sched *_lthread_sched_get(int lcore_id); 88 | 89 | struct lthread_stack *_stack_alloc(void); 90 | 91 | struct 92 | lthread_sched *_lthread_sched_create(unsigned lcore, size_t stack_size); 93 | 94 | void 95 | _lthread_init(struct lthread *lt, 96 | lthread_func_t fun, void *arg, lthread_exit_func exit_handler); 97 | 98 | void _lthread_set_stack(struct lthread *lt, void *stack, size_t stack_size); 99 | 100 | #endif /* LTHREAD_H_ */ 101 | -------------------------------------------------------------------------------- /mtcp/src/include/mtcp_api.h: -------------------------------------------------------------------------------- 1 | #ifndef __MTCP_API_H_ 2 | #define __MTCP_API_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #ifndef UNUSED 9 | #define UNUSED(x) (void)x 10 | #endif 11 | 12 | #ifndef INPORT_ANY 13 | #define INPORT_ANY (uint16_t)0 14 | #endif 15 | 16 | #ifdef __cplusplus 17 | extern "C" { 18 | #endif 19 | 20 | enum socket_type 21 | { 22 | MTCP_SOCK_UNUSED, 23 | MTCP_SOCK_STREAM, 24 | MTCP_SOCK_PROXY, 25 | MTCP_SOCK_LISTENER, 26 | MTCP_SOCK_EPOLL, 27 | MTCP_SOCK_PIPE, 28 | }; 29 | 30 | struct mtcp_conf 31 | { 32 | int num_cores; 33 | int max_concurrency; 34 | 35 | int max_num_buffers; 36 | int rcvbuf_size; 37 | int sndbuf_size; 38 | 39 | int tcp_timewait; 40 | int tcp_timeout; 41 | 42 | struct route_table *rtable; 43 | int routes; 44 | }; 45 | 46 | typedef struct mtcp_context *mctx_t; 47 | int 48 | mtcp_init(const char *config_file); 49 | 50 | void 51 | mtcp_destroy(); 52 | 53 | int 54 | mtcp_getconf(struct mtcp_conf *conf); 55 | 56 | int 57 | mtcp_setconf(const struct mtcp_conf *conf); 58 | 59 | int 60 | mtcp_core_affinitize(int cpu); 61 | 62 | mctx_t 63 | mtcp_create_context(int cpu); 64 | 65 | #ifdef ENABLE_UCTX 66 | typedef void (*mtcp_app_func_t) (void *); 67 | int 68 | mtcp_create_app_context(mctx_t mctx, mtcp_app_func_t func, void *arg); 69 | 70 | void 71 | mtcp_run_app(); 72 | #endif 73 | void 74 | mtcp_destroy_context(mctx_t mctx); 75 | 76 | typedef void (*mtcp_sighandler_t)(int); 77 | 78 | mtcp_sighandler_t 79 | mtcp_register_signal(int signum, mtcp_sighandler_t handler); 80 | 81 | int 82 | mtcp_pipe(mctx_t mctx, int pipeid[2]); 83 | 84 | int 85 | mtcp_getsockopt(mctx_t mctx, int sockid, int level, 86 | int optname, void *optval, socklen_t *optlen); 87 | 88 | int 89 | mtcp_setsockopt(mctx_t mctx, int sockid, int level, 90 | int optname, const void *optval, socklen_t optlen); 91 | 92 | int 93 | mtcp_setsock_nonblock(mctx_t mctx, int sockid); 94 | 95 | /* mtcp_socket_ioctl: similar to ioctl, 96 | but only FIONREAD is supported currently */ 97 | int 98 | mtcp_socket_ioctl(mctx_t mctx, int sockid, int request, void *argp); 99 | 100 | int 101 | mtcp_socket(mctx_t mctx, int domain, int type, int protocol); 102 | 103 | int 104 | mtcp_bind(mctx_t mctx, int sockid, 105 | const struct sockaddr *addr, socklen_t addrlen); 106 | 107 | int 108 | mtcp_listen(mctx_t mctx, int sockid, int backlog); 109 | 110 | int 111 | mtcp_accept(mctx_t mctx, int sockid, struct sockaddr *addr, socklen_t *addrlen); 112 | 113 | int 114 | mtcp_init_rss(mctx_t mctx, in_addr_t saddr_base, int num_addr, 115 | in_addr_t daddr, in_addr_t dport); 116 | 117 | int 118 | mtcp_connect(mctx_t mctx, int sockid, 119 | const struct sockaddr *addr, socklen_t addrlen); 120 | 121 | int 122 | mtcp_close(mctx_t mctx, int sockid); 123 | 124 | /** Returns the current address to which the socket sockfd is bound 125 | * @param [in] mctx: mtcp context 126 | * @param [in] addr: address buffer to be filled 127 | * @param [in] addrlen: amount of space pointed to by addr 128 | * @return 0 on success, -1 on error 129 | */ 130 | int 131 | mtcp_getsockname(mctx_t mctx, int sock, struct sockaddr *addr, socklen_t *addrlen); 132 | 133 | int 134 | mtcp_getpeername(mctx_t mctx, int sockid, struct sockaddr *addr, 135 | socklen_t *addrlen); 136 | 137 | //inline ssize_t 138 | ssize_t 139 | mtcp_read(mctx_t mctx, int sockid, char *buf, size_t len); 140 | 141 | ssize_t 142 | mtcp_recv(mctx_t mctx, int sockid, char *buf, size_t len, int flags); 143 | 144 | /* readv should work in atomic */ 145 | int 146 | mtcp_readv(mctx_t mctx, int sockid, const struct iovec *iov, int numIOV); 147 | 148 | ssize_t 149 | mtcp_write(mctx_t mctx, int sockid, const char *buf, size_t len); 150 | 151 | /* writev should work in atomic */ 152 | int 153 | mtcp_writev(mctx_t mctx, int sockid, const struct iovec *iov, int numIOV); 154 | 155 | #define TCP_SETUP_OFFLOAD 0x10 156 | #define TCP_TEARDOWN_OFFLOAD 0x11 157 | 158 | typedef struct nsplice_meta 159 | { 160 | uint32_t ip_addr; /* the ip address of client-side spliced connection */ 161 | uint16_t port; /* the port of client-side spliced connection */ 162 | uint32_t dst_addr; 163 | uint16_t dst_port; 164 | uint16_t close_reason; /* the reason of spliced connection close */ 165 | } nsplice_meta_t; 166 | 167 | typedef void (*callback_t)(nsplice_meta_t *meta); 168 | int 169 | mtcp_splice(mctx_t mctx, int sock_from, int sock_to, callback_t cb); 170 | 171 | #ifdef __cplusplus 172 | }; 173 | #endif 174 | 175 | #endif /* __MTCP_API_H_ */ 176 | -------------------------------------------------------------------------------- /mtcp/src/intel_lthread/lthread_diag.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * BSD LICENSE 3 | * 4 | * Copyright(c) 2015 Intel Corporation. All rights reserved. 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in 15 | * the documentation and/or other materials provided with the 16 | * distribution. 17 | * * Neither the name of Intel Corporation nor the names of its 18 | * contributors may be used to endorse or promote products derived 19 | * from this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | */ 33 | 34 | #ifndef LTHREAD_DIAG_H_ 35 | #define LTHREAD_DIAG_H_ 36 | 37 | #include 38 | #include 39 | 40 | #include 41 | #include 42 | 43 | #include "lthread_api.h" 44 | #include "lthread_diag_api.h" 45 | 46 | extern diag_callback diag_cb; 47 | 48 | extern const char *diag_event_text[]; 49 | extern uint64_t diag_mask; 50 | 51 | /* max size of name strings */ 52 | #define LT_MAX_NAME_SIZE 64 53 | 54 | #if LTHREAD_DIAG 55 | #define DISPLAY_OBJCACHE_QUEUES 1 56 | 57 | /* 58 | * Generate a diagnostic trace or event in the case where an object is created. 59 | * 60 | * The value returned by the callback is stored in the object. 61 | * 62 | * @ param obj 63 | * pointer to the object that was created 64 | * @ param ev 65 | * the event code 66 | * 67 | */ 68 | #define DIAG_CREATE_EVENT(obj, ev) do { \ 69 | struct lthread *ct = RTE_PER_LCORE(this_sched)->current_lthread;\ 70 | if ((BIT(ev) & diag_mask) && (ev < LT_DIAG_EVENT_MAX)) { \ 71 | (obj)->diag_ref = (diag_cb)(rte_rdtsc(), \ 72 | ct, \ 73 | (ev), \ 74 | 0, \ 75 | diag_event_text[(ev)], \ 76 | (uint64_t)obj, \ 77 | 0); \ 78 | } \ 79 | } while (0) 80 | 81 | /* 82 | * Generate a diagnostic trace event. 83 | * 84 | * @ param obj 85 | * pointer to the lthread, cond or mutex object 86 | * @ param ev 87 | * the event code 88 | * @ param p1 89 | * object specific value ( see lthread_diag_api.h ) 90 | * @ param p2 91 | * object specific value ( see lthread_diag_api.h ) 92 | */ 93 | #define DIAG_EVENT(obj, ev, p1, p2) do { \ 94 | struct lthread *ct = RTE_PER_LCORE(this_sched)->current_lthread;\ 95 | if ((BIT(ev) & diag_mask) && (ev < LT_DIAG_EVENT_MAX)) { \ 96 | (diag_cb)(rte_rdtsc(), \ 97 | ct, \ 98 | ev, \ 99 | (obj)->diag_ref, \ 100 | diag_event_text[(ev)], \ 101 | (uint64_t)(p1), \ 102 | (uint64_t)(p2)); \ 103 | } \ 104 | } while (0) 105 | 106 | #define DIAG_COUNT_DEFINE(x) rte_atomic64_t count_##x 107 | #define DIAG_COUNT_INIT(o, x) rte_atomic64_init(&((o)->count_##x)) 108 | #define DIAG_COUNT_INC(o, x) rte_atomic64_inc(&((o)->count_##x)) 109 | #define DIAG_COUNT_DEC(o, x) rte_atomic64_dec(&((o)->count_##x)) 110 | #define DIAG_COUNT(o, x) rte_atomic64_read(&((o)->count_##x)) 111 | 112 | #define DIAG_USED 113 | 114 | #else 115 | 116 | /* no diagnostics configured */ 117 | 118 | #define DISPLAY_OBJCACHE_QUEUES 0 119 | 120 | #define DIAG_CREATE_EVENT(obj, ev) 121 | #define DIAG_EVENT(obj, ev, p1, p) 122 | 123 | #define DIAG_COUNT_DEFINE(x) 124 | #define DIAG_COUNT_INIT(o, x) do {} while (0) 125 | #define DIAG_COUNT_INC(o, x) do {} while (0) 126 | #define DIAG_COUNT_DEC(o, x) do {} while (0) 127 | #define DIAG_COUNT(o, x) 0 128 | 129 | #define DIAG_USED __rte_unused 130 | 131 | #endif /* LTHREAD_DIAG */ 132 | #endif /* LTHREAD_DIAG_H_ */ 133 | -------------------------------------------------------------------------------- /mtcp/src/intel_lthread/lthread_objcache.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * BSD LICENSE 3 | * 4 | * Copyright(c) 2015 Intel Corporation. All rights reserved. 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in 15 | * the documentation and/or other materials provided with the 16 | * distribution. 17 | * * Neither the name of Intel Corporation nor the names of its 18 | * contributors may be used to endorse or promote products derived 19 | * from this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | */ 33 | #ifndef LTHREAD_OBJCACHE_H_ 34 | #define LTHREAD_OBJCACHE_H_ 35 | 36 | #include 37 | 38 | #include 39 | #include 40 | #include 41 | 42 | #include "lthread_int.h" 43 | #include "lthread_diag.h" 44 | #include "lthread_queue.h" 45 | 46 | 47 | RTE_DECLARE_PER_LCORE(struct lthread_sched *, this_sched); 48 | 49 | struct lthread_objcache { 50 | struct lthread_queue *q; 51 | size_t obj_size; 52 | int prealloc_size; 53 | char name[LT_MAX_NAME_SIZE]; 54 | 55 | DIAG_COUNT_DEFINE(rd); 56 | DIAG_COUNT_DEFINE(wr); 57 | DIAG_COUNT_DEFINE(prealloc); 58 | DIAG_COUNT_DEFINE(capacity); 59 | DIAG_COUNT_DEFINE(available); 60 | }; 61 | 62 | /* 63 | * Create a cache 64 | */ 65 | static inline struct 66 | lthread_objcache *_lthread_objcache_create(const char *name, 67 | size_t obj_size, 68 | int prealloc_size) 69 | { 70 | struct lthread_objcache *c = 71 | rte_malloc_socket(NULL, sizeof(struct lthread_objcache), 72 | RTE_CACHE_LINE_SIZE, 73 | rte_socket_id()); 74 | if (c == NULL) 75 | return NULL; 76 | 77 | c->q = _lthread_queue_create("cache queue"); 78 | if (c->q == NULL) { 79 | rte_free(c); 80 | return NULL; 81 | } 82 | c->obj_size = obj_size; 83 | c->prealloc_size = prealloc_size; 84 | 85 | if (name != NULL) 86 | strncpy(c->name, name, LT_MAX_NAME_SIZE); 87 | c->name[sizeof(c->name)-1] = 0; 88 | 89 | DIAG_COUNT_INIT(c, rd); 90 | DIAG_COUNT_INIT(c, wr); 91 | DIAG_COUNT_INIT(c, prealloc); 92 | DIAG_COUNT_INIT(c, capacity); 93 | DIAG_COUNT_INIT(c, available); 94 | return c; 95 | } 96 | 97 | /* 98 | * Destroy an objcache 99 | */ 100 | static inline int 101 | _lthread_objcache_destroy(struct lthread_objcache *c) 102 | { 103 | if (_lthread_queue_destroy(c->q) == 0) { 104 | rte_free(c); 105 | return 0; 106 | } 107 | return -1; 108 | } 109 | 110 | /* 111 | * Allocate an object from an object cache 112 | */ 113 | static inline void * 114 | _lthread_objcache_alloc(struct lthread_objcache *c) 115 | { 116 | int i; 117 | void *data; 118 | struct lthread_queue *q = c->q; 119 | size_t obj_size = c->obj_size; 120 | int prealloc_size = c->prealloc_size; 121 | 122 | data = _lthread_queue_remove(q); 123 | 124 | if (data == NULL) { 125 | DIAG_COUNT_INC(c, prealloc); 126 | for (i = 0; i < prealloc_size; i++) { 127 | data = 128 | rte_zmalloc_socket(NULL, obj_size, 129 | RTE_CACHE_LINE_SIZE, 130 | rte_socket_id()); 131 | if (data == NULL) 132 | return NULL; 133 | 134 | DIAG_COUNT_INC(c, available); 135 | DIAG_COUNT_INC(c, capacity); 136 | _lthread_queue_insert_mp(q, data); 137 | } 138 | data = _lthread_queue_remove(q); 139 | } 140 | DIAG_COUNT_INC(c, rd); 141 | DIAG_COUNT_DEC(c, available); 142 | return data; 143 | } 144 | 145 | /* 146 | * free an object to a cache 147 | */ 148 | static inline void 149 | _lthread_objcache_free(struct lthread_objcache *c, void *obj) 150 | { 151 | DIAG_COUNT_INC(c, wr); 152 | DIAG_COUNT_INC(c, available); 153 | _lthread_queue_insert_mp(c->q, obj); 154 | } 155 | 156 | 157 | 158 | #endif /* LTHREAD_OBJCACHE_H_ */ 159 | -------------------------------------------------------------------------------- /mtcp/src/include/io_module.h: -------------------------------------------------------------------------------- 1 | #ifndef __IO_MODULE_H__ 2 | #define __IO_MODULE_H__ 3 | /*----------------------------------------------------------------------------*/ 4 | /* for type def'ns */ 5 | #include 6 | #include 7 | #ifndef DISABLE_PSIO 8 | /* for ps lib funcs */ 9 | #include "ps.h" 10 | #endif 11 | 12 | #ifndef IFNAMSIZ 13 | #define IFNAMSIZ 16 14 | #endif 15 | 16 | #ifndef ETH_ALEN 17 | #define ETH_ALEN 6 18 | #endif 19 | 20 | #ifndef MAX_DEVICES 21 | #define MAX_DEVICES 16 22 | #endif 23 | /*----------------------------------------------------------------------------*/ 24 | /** 25 | * Declaration to soothe down the warnings 26 | */ 27 | struct mtcp_thread_context; 28 | /** 29 | * io_module_funcs - contains template for the various 10Gbps pkt I/O 30 | * - libraries that can be adopted. 31 | * 32 | * load_module() : Used to set system-wide I/O module 33 | * initialization. 34 | * 35 | * init_handle() : Used to initialize the driver library 36 | * : Also use the context to create/initialize 37 | * : a private packet I/O data structures. 38 | * 39 | * link_devices() : Used to add link(s) to the mtcp stack. 40 | * Returns 0 on success; -1 on failure. 41 | * 42 | * release_pkt() : release the packet if mTCP does not need 43 | * to process it (e.g. non-IPv4, non-TCP pkts). 44 | * 45 | * get_wptr() : retrieve the next empty pkt buffer for the 46 | * application for packet writing. Returns 47 | * ptr to pkt buffer. 48 | * 49 | * send_pkts() : transmit batch of packets via interface 50 | * idx (=nif). 51 | * Returns 0 on success; -1 on failure 52 | * 53 | * get_rptr() : retrieve next pkt for application for 54 | * packet read. 55 | * Returns ptr to pkt buffer. 56 | * 57 | * recv_pkts() : recieve batch of packets from the interface, 58 | * ifidx. 59 | * Returns no. of packets that are read from 60 | * the iface. 61 | * 62 | * select() : for blocking I/O 63 | * 64 | * destroy_handle() : free up resources allocated during 65 | * init_handle(). Normally called during 66 | * process termination. 67 | * 68 | * dev_ioctl() : contains submodules for select drivers 69 | * 70 | */ 71 | typedef struct io_module_func { 72 | void (*load_module)(void); 73 | void (*init_handle)(struct mtcp_thread_context *ctx); 74 | int32_t (*link_devices)(struct mtcp_thread_context *ctx); 75 | void (*release_pkt)(struct mtcp_thread_context *ctx, int ifidx, unsigned char *pkt_data, int len); 76 | uint8_t * (*get_wptr)(struct mtcp_thread_context *ctx, int ifidx, uint16_t len); 77 | int32_t (*send_pkts)(struct mtcp_thread_context *ctx, int nif); 78 | uint8_t * (*get_rptr)(struct mtcp_thread_context *ctx, int ifidx, int index, uint16_t *len); 79 | int32_t (*recv_pkts)(struct mtcp_thread_context *ctx, int ifidx); 80 | int32_t (*select)(struct mtcp_thread_context *ctx); 81 | void (*destroy_handle)(struct mtcp_thread_context *ctx); 82 | int32_t (*dev_ioctl)(struct mtcp_thread_context *ctx, int nif, int cmd, void *argp); 83 | } io_module_func __attribute__((aligned(__WORDSIZE))); 84 | /*----------------------------------------------------------------------------*/ 85 | /* set I/O module context */ 86 | int SetInterfaceInfo(char *); 87 | 88 | /* retrive device-specific endian type */ 89 | int FetchEndianType(); 90 | /*----------------------------------------------------------------------------*/ 91 | /* ptr to the `running' I/O module context */ 92 | extern io_module_func *current_iomodule_func; 93 | 94 | /* dev_ioctl related macros */ 95 | #define PKT_TX_IP_CSUM 0x01 96 | #define PKT_TX_TCP_CSUM 0x02 97 | #define PKT_RX_TCP_LROSEG 0x03 98 | #define PKT_TX_TCPIP_CSUM 0x04 99 | #define PKT_RX_IP_CSUM 0x05 100 | #define PKT_RX_TCP_CSUM 0x06 101 | #define PKT_TX_TCPIP_CSUM_PEEK 0x07 102 | #define DRV_NAME 0x08 103 | 104 | #ifdef ENABLE_APP_INTERRUPT 105 | #define DRV_PENDING_RX_PKT 0x0B 106 | #endif /* ENABLE_APP_INTERRUPT */ 107 | 108 | /* registered psio context */ 109 | #ifdef DISABLE_PSIO 110 | #define ps_list_devices(x) 0 111 | #else 112 | struct ps_device devices[MAX_DEVICES]; 113 | #endif 114 | 115 | extern io_module_func ps_module_func; 116 | 117 | /* registered dpdk context */ 118 | extern io_module_func dpdk_module_func; 119 | 120 | /* registered netmap context */ 121 | extern io_module_func netmap_module_func; 122 | 123 | /* Macro to assign IO module */ 124 | #define AssignIOModule(m) { \ 125 | if (!strcmp(m, "psio")) \ 126 | current_iomodule_func = &ps_module_func; \ 127 | else if (!strcmp(m, "dpdk")) \ 128 | current_iomodule_func = &dpdk_module_func; \ 129 | else if (!strcmp(m, "netmap")) \ 130 | current_iomodule_func = &netmap_module_func; \ 131 | else \ 132 | assert(0); \ 133 | } 134 | 135 | /* XXX temporary fix for XL710 NICs that does not support ntuple */ 136 | #define XL710 137 | 138 | /*----------------------------------------------------------------------------*/ 139 | #endif /* !__IO_MODULE_H__ */ 140 | -------------------------------------------------------------------------------- /mtcp/src/ip_out.c: -------------------------------------------------------------------------------- 1 | #include "ip_out.h" 2 | #include "ip_in.h" 3 | #include "eth_out.h" 4 | #include "arp.h" 5 | #include "debug.h" 6 | 7 | /*----------------------------------------------------------------------------*/ 8 | inline int 9 | GetOutputInterface(uint32_t daddr, uint8_t* is_external) 10 | { 11 | int nif = -1; 12 | int i; 13 | int prefix = 0; 14 | 15 | *is_external = 0; 16 | /* Longest prefix matching */ 17 | for (i = 0; i < CONFIG.routes; i++) { 18 | if ((daddr & CONFIG.rtable[i].mask) == CONFIG.rtable[i].masked) { 19 | if (CONFIG.rtable[i].prefix > prefix) { 20 | nif = CONFIG.rtable[i].nif; 21 | prefix = CONFIG.rtable[i].prefix; 22 | } 23 | else if (CONFIG.gateway) { 24 | *is_external = 1; 25 | nif = (CONFIG.gateway)->nif; 26 | } 27 | break; 28 | } 29 | } 30 | 31 | if (nif < 0) { 32 | uint8_t *da = (uint8_t *)&daddr; 33 | TRACE_ERROR("[WARNING] No route to %u.%u.%u.%u\n", 34 | da[0], da[1], da[2], da[3]); 35 | assert(0); 36 | } 37 | 38 | return nif; 39 | } 40 | /*----------------------------------------------------------------------------*/ 41 | uint8_t * 42 | IPOutputStandalone(struct mtcp_manager *mtcp, uint16_t eth_type, uint8_t protocol, 43 | uint16_t ip_id, uint32_t saddr, uint32_t daddr, uint16_t payloadlen) 44 | { 45 | struct iphdr *iph; 46 | int nif; 47 | unsigned char * haddr, is_external; 48 | int rc = -1; 49 | 50 | nif = GetOutputInterface(daddr, &is_external); 51 | if (nif < 0) 52 | return NULL; 53 | 54 | haddr = GetDestinationHWaddr(daddr, is_external); 55 | if (!haddr) { 56 | #if 1 57 | uint8_t *da = (uint8_t *)&daddr; 58 | TRACE_INFO("[WARNING] The destination IP %u.%u.%u.%u " 59 | "is not in ARP table!\n", 60 | da[0], da[1], da[2], da[3]); 61 | #endif 62 | RequestARP(mtcp, daddr, nif, mtcp->cur_ts); 63 | return NULL; 64 | } 65 | 66 | iph = (struct iphdr *)EthernetOutput(mtcp, 67 | eth_type, nif, haddr, payloadlen + IP_HEADER_LEN, 68 | NULL); 69 | if (!iph) { 70 | return NULL; 71 | } 72 | 73 | iph->ihl = IP_HEADER_LEN >> 2; 74 | iph->version = 4; 75 | iph->tos = 0; 76 | iph->tot_len = htons(IP_HEADER_LEN + payloadlen); 77 | iph->id = htons(ip_id); 78 | iph->frag_off = htons(IP_DF); // no fragmentation 79 | iph->ttl = 64; 80 | iph->protocol = protocol; 81 | iph->saddr = saddr; 82 | iph->daddr = daddr; 83 | iph->check = 0; 84 | 85 | #ifndef DISABLE_HWCSUM 86 | if (mtcp->iom->dev_ioctl != NULL) { 87 | switch(iph->protocol) { 88 | case IPPROTO_TCP: 89 | rc = mtcp->iom->dev_ioctl(mtcp->ctx, nif, PKT_TX_TCPIP_CSUM_PEEK, iph); 90 | break; 91 | case IPPROTO_ICMP: 92 | rc = mtcp->iom->dev_ioctl(mtcp->ctx, nif, PKT_TX_IP_CSUM, iph); 93 | break; 94 | } 95 | } 96 | /* otherwise calculate IP checksum in S/W */ 97 | if (rc == -1) 98 | iph->check = ip_fast_csum(iph, iph->ihl); 99 | #else 100 | UNUSED(rc); 101 | iph->check = ip_fast_csum(iph, iph->ihl); 102 | #endif 103 | 104 | return (uint8_t *)(iph + 1); 105 | } 106 | /*----------------------------------------------------------------------------*/ 107 | uint8_t * 108 | IPOutput(struct mtcp_manager *mtcp, tcp_stream *stream, uint16_t tcplen, 109 | struct mtcp_offload_meta* offload_meta) 110 | { 111 | struct iphdr *iph; 112 | int nif; 113 | unsigned char *haddr, is_external = 0; 114 | int rc = -1; 115 | 116 | if (stream->sndvar->nif_out >= 0) { 117 | nif = stream->sndvar->nif_out; 118 | } else { 119 | nif = GetOutputInterface(stream->daddr, &is_external); 120 | stream->sndvar->nif_out = nif; 121 | } 122 | 123 | haddr = GetDestinationHWaddr(stream->daddr, stream->is_external); 124 | if (!haddr) { 125 | #if 0 126 | uint8_t *da = (uint8_t *)&stream->daddr; 127 | TRACE_INFO("[WARNING] The destination IP %u.%u.%u.%u " 128 | "is not in ARP table!\n", 129 | da[0], da[1], da[2], da[3]); 130 | #endif 131 | /* if not found in the arp table, send arp request and return NULL */ 132 | /* tcp will retry sending the packet later */ 133 | RequestARP(mtcp, stream->daddr, stream->sndvar->nif_out, mtcp->cur_ts); 134 | return NULL; 135 | } 136 | 137 | iph = (struct iphdr *)EthernetOutput(mtcp, ETH_P_IP, 138 | stream->sndvar->nif_out, haddr, tcplen + IP_HEADER_LEN, 139 | offload_meta); 140 | if (!iph) { 141 | return NULL; 142 | } 143 | 144 | iph->ihl = IP_HEADER_LEN >> 2; 145 | iph->version = 4; 146 | iph->tos = 0; 147 | iph->tot_len = htons(IP_HEADER_LEN + tcplen); 148 | iph->id = htons(stream->sndvar->ip_id++); 149 | iph->frag_off = htons(0x4000); // no fragmentation 150 | iph->ttl = 64; 151 | iph->protocol = IPPROTO_TCP; 152 | iph->saddr = stream->saddr; 153 | iph->daddr = stream->daddr; 154 | iph->check = 0; 155 | 156 | #ifndef DISABLE_HWCSUM 157 | /* offload IP checkum if possible */ 158 | if (mtcp->iom->dev_ioctl != NULL) { 159 | switch (iph->protocol) { 160 | case IPPROTO_TCP: 161 | rc = mtcp->iom->dev_ioctl(mtcp->ctx, nif, PKT_TX_TCPIP_CSUM_PEEK, iph); 162 | break; 163 | case IPPROTO_ICMP: 164 | rc = mtcp->iom->dev_ioctl(mtcp->ctx, nif, PKT_TX_IP_CSUM, iph); 165 | break; 166 | } 167 | } 168 | /* otherwise calculate IP checksum in S/W */ 169 | if (rc == -1) 170 | iph->check = ip_fast_csum(iph, iph->ihl); 171 | #else 172 | UNUSED(rc); 173 | iph->check = ip_fast_csum(iph, iph->ihl); 174 | #endif 175 | return (uint8_t *)(iph + 1); 176 | } 177 | -------------------------------------------------------------------------------- /mtcp/src/icmp.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "mtcp.h" 6 | #include "icmp.h" 7 | #include "eth_out.h" 8 | #include "ip_in.h" 9 | #include "ip_out.h" 10 | #include "debug.h" 11 | #include "arp.h" 12 | 13 | #define IP_NEXT_PTR(iph) ((uint8_t *)iph + (iph->ihl << 2)) 14 | /*----------------------------------------------------------------------------*/ 15 | void 16 | DumpICMPPacket(struct icmphdr *icmph, uint32_t saddr, uint32_t daddr); 17 | /*----------------------------------------------------------------------------*/ 18 | static uint16_t 19 | ICMPChecksum(uint16_t *icmph, int len) 20 | { 21 | assert(len >= 0); 22 | 23 | uint16_t ret = 0; 24 | uint32_t sum = 0; 25 | uint16_t odd_byte; 26 | 27 | while (len > 1) { 28 | sum += *icmph++; 29 | len -= 2; 30 | } 31 | 32 | if (len == 1) { 33 | *(uint8_t*)(&odd_byte) = * (uint8_t*)icmph; 34 | sum += odd_byte; 35 | } 36 | 37 | sum = (sum >> 16) + (sum & 0xffff); 38 | sum += (sum >> 16); 39 | ret = ~sum; 40 | 41 | return ret; 42 | } 43 | /*----------------------------------------------------------------------------*/ 44 | static int 45 | ICMPOutput(struct mtcp_manager *mtcp, uint32_t saddr, uint32_t daddr, 46 | uint8_t icmp_type, uint8_t icmp_code, uint16_t icmp_id, uint16_t icmp_seq, 47 | uint8_t *icmpd, uint16_t len) 48 | { 49 | struct icmphdr *icmph; 50 | 51 | icmph = (struct icmphdr *)IPOutputStandalone(mtcp, ETH_P_IP, 52 | IPPROTO_ICMP, 0, saddr, daddr, sizeof(struct icmphdr) + len); 53 | if (!icmph) 54 | return -1; 55 | 56 | /* Fill in the icmp header */ 57 | icmph->icmp_type = icmp_type; 58 | icmph->icmp_code = icmp_code; 59 | icmph->icmp_checksum = 0; 60 | ICMP_ECHO_SET_ID(icmph, htons(icmp_id)); 61 | ICMP_ECHO_SET_SEQ(icmph, htons(icmp_seq)); 62 | 63 | /* Fill in the icmp data */ 64 | if (len > 0) 65 | memcpy((void *)(icmph + 1), icmpd, len); 66 | 67 | /* Calculate ICMP Checksum with header and data */ 68 | icmph->icmp_checksum = 69 | ICMPChecksum((uint16_t *)icmph, sizeof(struct icmphdr) + len); 70 | 71 | #if DBGMSG 72 | DumpICMPPacket(icmph, saddr, daddr); 73 | #endif 74 | return 0; 75 | } 76 | /*----------------------------------------------------------------------------*/ 77 | void 78 | RequestICMP(mtcp_manager_t mtcp, uint32_t saddr, uint32_t daddr, 79 | uint16_t icmp_id, uint16_t icmp_sequence, 80 | uint8_t *icmpd, uint16_t len) 81 | { 82 | /* send icmp request with given parameters */ 83 | ICMPOutput(mtcp, saddr, daddr, ICMP_ECHO, 0, ntohs(icmp_id), ntohs(icmp_sequence), 84 | icmpd, len); 85 | } 86 | /*----------------------------------------------------------------------------*/ 87 | static int 88 | ProcessICMPECHORequest(mtcp_manager_t mtcp, struct iphdr *iph, int len) 89 | { 90 | int ret = 0; 91 | struct icmphdr *icmph = (struct icmphdr *) IP_NEXT_PTR(iph); 92 | 93 | /* Check correctness of ICMP checksum and send ICMP echo reply */ 94 | if (ICMPChecksum((uint16_t *) icmph, len - (iph->ihl << 2)) ) 95 | ret = ERROR; 96 | else 97 | ICMPOutput(mtcp, iph->daddr, iph->saddr, ICMP_ECHOREPLY, 0, 98 | ntohs(ICMP_ECHO_GET_ID(icmph)), ntohs(ICMP_ECHO_GET_SEQ(icmph)), 99 | (uint8_t *) (icmph + 1), 100 | (uint16_t) (len - (iph->ihl << 2) - sizeof(struct icmphdr)) ); 101 | 102 | return ret; 103 | } 104 | /*----------------------------------------------------------------------------*/ 105 | int 106 | ProcessICMPPacket(mtcp_manager_t mtcp, struct iphdr *iph, int len) 107 | { 108 | struct icmphdr *icmph = (struct icmphdr *) IP_NEXT_PTR(iph); 109 | int i; 110 | int to_me = FALSE; 111 | 112 | /* process the icmp messages destined to me */ 113 | for (i = 0; i < CONFIG.eths_num; i++) { 114 | if (iph->daddr == CONFIG.eths[i].ip_addr) { 115 | to_me = TRUE; 116 | } 117 | } 118 | 119 | if (!to_me) 120 | return TRUE; 121 | 122 | switch (icmph->icmp_type) { 123 | case ICMP_ECHO: 124 | ProcessICMPECHORequest(mtcp, iph, len); 125 | break; 126 | 127 | case ICMP_DEST_UNREACH: 128 | TRACE_INFO("[INFO] ICMP Destination Unreachable message received\n"); 129 | break; 130 | 131 | case ICMP_TIME_EXCEEDED: 132 | TRACE_INFO("[INFO] ICMP Time Exceeded message received\n"); 133 | break; 134 | 135 | default: 136 | TRACE_INFO("[INFO] Unsupported ICMP message type %x received\n", 137 | icmph->icmp_type); 138 | break; 139 | } 140 | 141 | return TRUE; 142 | } 143 | /*----------------------------------------------------------------------------*/ 144 | void 145 | DumpICMPPacket(struct icmphdr *icmph, uint32_t saddr, uint32_t daddr) 146 | { 147 | uint8_t *t; 148 | 149 | fprintf(stderr, "ICMP header: \n"); 150 | fprintf(stderr, "Type: %d, " 151 | "Code: %d, ID: %d, Sequence: %d\n", 152 | icmph->icmp_type, icmph->icmp_code, 153 | ntohs(ICMP_ECHO_GET_ID(icmph)), ntohs(ICMP_ECHO_GET_SEQ(icmph))); 154 | 155 | t = (uint8_t *)&saddr; 156 | fprintf(stderr, "Sender IP: %u.%u.%u.%u\n", 157 | t[0], t[1], t[2], t[3]); 158 | 159 | t = (uint8_t *)&daddr; 160 | fprintf(stderr, "Target IP: %u.%u.%u.%u\n", 161 | t[0], t[1], t[2], t[3]); 162 | } 163 | /*----------------------------------------------------------------------------*/ 164 | #undef IP_NEXT_PTR 165 | -------------------------------------------------------------------------------- /nic/setup_teardown/src/include/conn_table.h: -------------------------------------------------------------------------------- 1 | #ifndef STATE_TABLE_H 2 | #define STATE_TABLE_H 3 | /*------------------------------------------------------------------------*/ 4 | #include 5 | /*------------------------------------------------------------------------*/ 6 | #define FLOWS_PER_GROUP 64 /* RTO bits are grouped 4 (uint64_t) */ 7 | #define BITSHIFT_GROUP 6 /* RTO bits are grouped by 2 ^ 6 = 64 (uint64_t) */ 8 | #define MAX_FLOW_GROUP (MAX_FLOWS >> BITSHIFT_GROUP) 9 | #define FLOW_GROUP_PER_ME 51 /* FIXME: hard-coded (# MES is not power of 2) */ 10 | /*------------------------------------------------------------------------*/ 11 | /* calculate the flow id with addr and port */ 12 | #define FLOWID(addr, port) (((addr) & (NUM_HOSTS - 1)) << 16) | (port) 13 | /* convert the flow id to a RTO bit */ 14 | #define FLOWID_TO_BITS(fid) ((uint64_t) 1 << (fid & (FLOWS_PER_GROUP - 1))) 15 | /* retrieve the length of the offloaded packet of the flow id */ 16 | #define FLOWID_TO_PKTCNT(fid) ((g_flow_table[fid].pktCnt_pktAcked_rtoGroup & 0xF00) >> 8) 17 | /* retrieve the length of the offloaded packet of the flow id */ 18 | #define FLOWID_TO_PKTACKED(fid) ((g_flow_table[fid].pktCnt_pktAcked_rtoGroup & 0xF0) >> 4) 19 | /* retrieve which RTO group the flow id belongs to */ 20 | #define FLOWID_TO_RTOGROUP(fid) ((g_flow_table[fid].pktCnt_pktAcked_rtoGroup) \ 21 | & BITMASK_RTO_GROUPS) 22 | #define FLOWID_TO_TWCYCLE(fid) (g_flow_table[fid].pktCnt_pktAcked_rtoGroup >> 28) 23 | /*------------------------------------------------------------------------*/ 24 | typedef struct flow_entry { 25 | /* TODO: hold 5-tuple of each flow here in hash table version */ 26 | /* flow states */ 27 | /* from LSB: (11:8) = pktCnt, (7:4) = pktAcked, (3:0) = rtoGroup */ 28 | uint32_t pktCnt_pktAcked_rtoGroup; 29 | /* expected sequence number of incoming FIN packet */ 30 | uint32_t expectedSeqNo; 31 | /* expected acknowledge number of incoming ACK packet */ 32 | uint32_t expectedAckNo; 33 | } flow_entry; 34 | /*------------------------------------------------------------------------*/ 35 | typedef struct pktbuf { 36 | uint32_t buf[MAX_PKTNUM][PKTBUF_SIZE]; 37 | } pktbuf; 38 | /*------------------------------------------------------------------------*/ 39 | __declspec(imem export scope(global)) flow_entry g_flow_table[MAX_FLOWS]; 40 | __declspec(imem export scope(global)) uint64_t g_retx_map[NUM_RTO_GROUPS] \ 41 | [MAX_FLOW_GROUP]; 42 | __declspec(emem export scope(global)) pktbuf state_pktbuf[MAX_FLOWS]; 43 | /*------------------------------------------------------------------------*/ 44 | void 45 | table_init() { 46 | __xwrite uint32_t empty = 0; 47 | __xwrite uint64_t empty_64 = 0; 48 | int i, j; 49 | 50 | for (i = 0; i < MAX_FLOWS; i++) { 51 | mem_write32(&empty, &(g_flow_table[i].pktCnt_pktAcked_rtoGroup), sizeof(uint32_t)); 52 | } 53 | 54 | for (i = 0; i < NUM_RTO_GROUPS; i++) { 55 | for (j = 0; j < MAX_FLOW_GROUP; j++) { 56 | mem_write32(&empty_64, &(g_retx_map[i][j]), sizeof(uint64_t)); 57 | } 58 | } 59 | } 60 | /*------------------------------------------------------------------------*/ 61 | void 62 | table_insert(__xwrite flow_entry* insert_entry, uint32_t addr, 63 | uint16_t port, uint8_t rtoGroup) { 64 | 65 | uint32_t fid = FLOWID(addr, port); 66 | 67 | #if ENABLE_TEARDOWN_RTO 68 | uint32_t idx = fid >> BITSHIFT_GROUP; 69 | __xwrite uint64_t xwr = FLOWID_TO_BITS(fid); 70 | mem_bitset(&xwr, &g_retx_map[rtoGroup][idx], sizeof(uint64_t)); 71 | #endif 72 | 73 | mem_write32(insert_entry, &(g_flow_table[fid]), sizeof(flow_entry)); 74 | } 75 | /*------------------------------------------------------------------------*/ 76 | int 77 | table_lookup(uint32_t addr, uint16_t port) { 78 | 79 | return (g_flow_table[FLOWID(addr, port)].pktCnt_pktAcked_rtoGroup != 0); 80 | } 81 | /*------------------------------------------------------------------------*/ 82 | int 83 | table_lookup_and_fetch(uint32_t addr, uint16_t port, 84 | __xread flow_entry* read_entry) { 85 | 86 | mem_read32(read_entry, &(g_flow_table[FLOWID(addr, port)]), 87 | sizeof(flow_entry)); 88 | 89 | return (read_entry->pktCnt_pktAcked_rtoGroup != 0); 90 | } 91 | /*------------------------------------------------------------------------*/ 92 | #if ENABLE_TEARDOWN_TIMEWAIT 93 | void 94 | table_timewait(uint32_t addr, uint16_t port) { 95 | 96 | uint32_t fid = FLOWID(addr, port); 97 | uint32_t flow_state = g_flow_table[fid].pktCnt_pktAcked_rtoGroup; 98 | g_flow_table[fid].pktCnt_pktAcked_rtoGroup = (INIT_TWCYCLE | flow_state); 99 | } 100 | 101 | void 102 | table_timewait_decr(uint32_t fid) { 103 | uint32_t flow_state = g_flow_table[fid].pktCnt_pktAcked_rtoGroup; 104 | g_flow_table[fid].pktCnt_pktAcked_rtoGroup = (flow_state - TWCYCLE_1); 105 | } 106 | 107 | #endif 108 | /*------------------------------------------------------------------------*/ 109 | void 110 | table_remove_fid(uint32_t fid) { 111 | __xwrite uint32_t empty = 0; 112 | #if ENABLE_TEARDOWN_RTO 113 | uint8_t rtoGroup = FLOWID_TO_RTOGROUP(fid); 114 | uint32_t idx = fid >> BITSHIFT_GROUP; 115 | __xwrite uint64_t xwr = FLOWID_TO_BITS(fid); 116 | mem_bitclr(&xwr, &g_retx_map[rtoGroup][idx], sizeof(uint64_t)); 117 | #endif 118 | 119 | mem_write32(&empty, &(g_flow_table[fid].pktCnt_pktAcked_rtoGroup), 120 | sizeof(uint32_t)); 121 | } 122 | /*------------------------------------------------------------------------*/ 123 | #endif 124 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.69]) 5 | AC_INIT([mtcp], [3], [mtcp-user@list.ndsl.kaist.edu]) 6 | AC_CONFIG_SRCDIR([mtcp/src/eventpoll.c]) 7 | AC_CONFIG_HEADERS([config.h]) 8 | 9 | # Checks for programs. 10 | AC_PROG_CC 11 | AC_PROG_RANLIB 12 | AC_LANG(C) 13 | AC_DISABLE_OPTION_CHECKING 14 | 15 | # Checks for header files. 16 | AC_HEADER_STDC 17 | AC_CHECK_HEADERS([linux/sched.h pthread.h numa.h arpa/inet.h fcntl.h limits.h netdb.h netinet/in.h stdint.h stdlib.h string.h sys/ioctl.h sys/socket.h sys/time.h unistd.h]) 18 | 19 | # Check for AC_PTHREAD [FC is preventing this to be enabled..] 20 | #AX_PTHREAD( [ 21 | # AC_DEFINE(HAVE_PTHREAD,1,[Define if you have POSIX threads libraries and header files.]) 22 | # CLIBS="$PTHREAD_LIBS $LIBS" 23 | # CFLAGS="$CFLAGS $PTHREAD_CFLAGS" 24 | # LDFLAGS="$LDFLAGS $PTHREAD_CFLAGS" 25 | # CC="$PTHREAD_CC"],[]) 26 | 27 | # Checks for typedefs, structures, and compiler characteristics. 28 | AC_C_INLINE 29 | AC_TYPE_INT16_T 30 | AC_TYPE_INT32_T 31 | AC_TYPE_INT64_T 32 | AC_TYPE_INT8_T 33 | AC_TYPE_PID_T 34 | AC_TYPE_SIZE_T 35 | AC_TYPE_SSIZE_T 36 | AC_TYPE_UINT16_T 37 | AC_TYPE_UINT32_T 38 | AC_TYPE_UINT64_T 39 | AC_TYPE_UINT8_T 40 | 41 | # Checks for library functions. 42 | AC_FUNC_MALLOC 43 | AC_FUNC_MMAP 44 | AC_CHECK_FUNCS([bzero clock_gettime getpagesize gettimeofday memmove memset munmap select socket strchr strerror strstr strtol]) 45 | 46 | # Reset DPDK to 0 47 | AC_SUBST(DPDK, 0) 48 | # Reset DPDK's LRO to 0 49 | AC_SUBST(LRO, 0) 50 | # Reset PSIO to 0 51 | AC_SUBST(PSIO, 0) 52 | # Reset NETMAP to 0 53 | AC_SUBST(NETMAP, 0) 54 | # Reset HWCSUM to 1 55 | AC_SUBST(HWCSUM, 1) 56 | # Reset UCTX to 1 57 | AC_SUBST(UCTX, 1) 58 | # Reset VFIO to 0 59 | AC_SUBST(VFIO, 0) 60 | # Reset LOGGER to 0 61 | AC_SUBST(LOGGER, 0) 62 | # Reset APP_INTERRUPT to 0 63 | AC_SUBST(APP_INTERRUPT, 0) 64 | # Reset NFP to 0 65 | AC_SUBST(NFP, 0) 66 | 67 | # Check dpdk-17.08 path (lib & inc) 68 | AC_ARG_WITH(stuff, [ --with-dpdk-lib path to the dpdk-17.08 install root]) 69 | if test "$with_dpdk_lib" != "" 70 | then 71 | AC_SUBST(DPDKLIBPATH, $with_dpdk_lib) 72 | AC_SUBST(DPDK, 1) 73 | AM_CONDITIONAL([DPDK], [test x$DPDK = x1]) 74 | fi 75 | dnl Example of default-disabled feature 76 | AC_ARG_ENABLE([lro], 77 | AS_HELP_STRING([--enable-lro], [Enable dpdk lro (for relevant NICs)])) 78 | 79 | if test "$with_dpdk_lib" != "" 80 | then 81 | if test "x$enable_lro" = "xyes" 82 | then 83 | AC_SUBST(LRO, 1) 84 | fi 85 | fi 86 | 87 | dnl Example of default-enabled feature 88 | AC_ARG_ENABLE([hwcsum], 89 | AS_HELP_STRING([--disable-hwcsum], [Disable h/w-based checksum offloading (for relevant NICs)])) 90 | 91 | if test "x$enable_hwcsum" = "xno" 92 | then 93 | AC_SUBST(HWCSUM, 0) 94 | fi 95 | 96 | #AC_ARG_ENABLE([uctx], 97 | # AS_HELP_STRING([--enable-uctx], [Enable dpdk uctx (for relevant NICs)])) 98 | 99 | #if test "$with_dpdk_lib" != "" 100 | #then 101 | # if test "x$enable_uctx" = "xyes" 102 | # then 103 | # AC_SUBST(UCTX, 1) 104 | # fi 105 | #fi 106 | 107 | #AC_ARG_ENABLE([vfio], 108 | # AS_HELP_STRING([--enable-vfio], [Enable DPDK VFIO for Netronome Agilio (NFP-6xxx) NICs)])) 109 | 110 | #if test "$with_dpdk_lib" != "" 111 | #then 112 | # if test "x$enable_vfio" = "xyes" 113 | # then 114 | # AC_SUBST(VFIO, 1) 115 | # fi 116 | #fi 117 | 118 | #AC_ARG_ENABLE([per_core], 119 | # AS_HELP_STRING([--enable-percore], [Enable dpdk per core (for relevant NICs)])) 120 | 121 | #if test "$with_dpdk_lib" != "" 122 | #then 123 | # if test "x$enable_percore" = "xyes" 124 | # then 125 | # AC_SUBST(PER_CORE, 1) 126 | # fi 127 | #fi 128 | 129 | AC_ARG_ENABLE([logger], 130 | AS_HELP_STRING([--enable-logger], [Enable logger thread])) 131 | 132 | if test "x$enable_logger" = "xyes" 133 | then 134 | AC_SUBST(LOGGER, 1) 135 | fi 136 | 137 | AC_ARG_ENABLE([app_interrupt], 138 | AS_HELP_STRING([--enable-app-interrupt], [Enable interrupting application context])) 139 | 140 | if test "x$enable_uctx" = "xyes" 141 | then 142 | if test "x$enable_app_interrupt" = "xyes" 143 | then 144 | AC_SUBST(APP_INTERRUPT, 1) 145 | fi 146 | fi 147 | 148 | AC_ARG_ENABLE([nfp], 149 | AS_HELP_STRING([--enable-nfp], [Use Netronome Agilio NICs (e.g., NFP-6xxx)])) 150 | 151 | if test "$with_dpdk_lib" != "" 152 | then 153 | if test "x$enable_nfp" = "xyes" 154 | then 155 | AC_SUBST(VFIO, 1) 156 | AC_SUBST(HWCSUM, 0) 157 | AC_SUBST(NFP, 1) 158 | fi 159 | fi 160 | 161 | # Check psio path (lib & inc) 162 | #AC_ARG_WITH(stuff, [ --with-psio-lib path to the ioengine install root]) 163 | #if test "$with_psio_lib" != "" 164 | #then 165 | # AC_SUBST(PSLIBPATH, $with_psio_lib) 166 | # AC_SUBST(PSIO, 1) 167 | #fi 168 | 169 | dnl Example of default-disabled feature 170 | #AC_ARG_ENABLE([netmap], 171 | # AS_HELP_STRING([--enable-netmap], [Enable netmap module])) 172 | 173 | #AS_IF([test "x$enable_netmap" = "xyes"], [ 174 | # AC_SUBST(NETMAP, 1) 175 | #]) 176 | 177 | if test "$with_dpdk_lib" == "" 178 | then 179 | #AC_MSG_ERROR([Packet I/O library is missing. Please set either dpdk or psio or netmap as your I/O lib.]) 180 | AC_MSG_ERROR([Please set the path to your dpdk lib (./configure --with-dpdk-lib=[path_to_dpdk]).]) 181 | fi 182 | 183 | AM_INIT_AUTOMAKE(mtcp, 3) 184 | AC_CONFIG_FILES([Makefile]) 185 | AC_OUTPUT(mtcp/src/Makefile) 186 | AC_OUTPUT(apps/example/Makefile) 187 | AC_OUTPUT(apps/epproxy/Makefile) 188 | #AC_OUTPUT(io_engine/lib/Makefile) 189 | -------------------------------------------------------------------------------- /mtcp/src/include/debug.h: -------------------------------------------------------------------------------- 1 | #ifndef __DEBUG_H_ 2 | #define __DEBUG_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "mtcp.h" 8 | #include "tcp_in.h" 9 | 10 | #ifdef DBGTEMP 11 | 12 | #define TRACE_TEMP(f, m...) { \ 13 | fprintf(stderr, "[CPU %d][%10s:%4d] " f, mtcp->ctx->cpu, \ 14 | __FUNCTION__, __LINE__, ##m); \ 15 | } 16 | 17 | #else 18 | 19 | #define TRACE_TEMP(f, m...) (void)0 20 | 21 | #endif /* DBGTEMP*/ 22 | 23 | #ifdef DBGERR 24 | 25 | #define TRACE_ERROR(f, m...) { \ 26 | fprintf(stderr, "[%10s:%4d] " f, __FUNCTION__, __LINE__, ##m); \ 27 | } 28 | 29 | #else 30 | 31 | #define TRACE_ERROR(f, m...) (void)0 32 | 33 | #endif /* DBGERR */ 34 | 35 | #ifdef DBGCERR 36 | 37 | #define CTRACE_ERROR(f, m...) { \ 38 | fprintf(stderr, "[CPU %d][%10s:%4d] " f, mtcp->ctx->cpu, __FUNCTION__, __LINE__, ##m); \ 39 | } 40 | 41 | #else 42 | 43 | #define CTRACE_ERROR(f, m...) (void)0 44 | 45 | #endif /* DBGERR */ 46 | 47 | #ifdef DBGMSG 48 | 49 | #define TRACE_DBG(f, m...) {\ 50 | thread_printf(mtcp, mtcp->log_fp, "[%10s:%4d] " \ 51 | f, __FUNCTION__, __LINE__, ##m); \ 52 | } 53 | 54 | #else 55 | 56 | #define TRACE_DBG(f, m...) (void)0 57 | 58 | #endif /* DBGMSG */ 59 | 60 | #ifdef INFO 61 | 62 | #define TRACE_INFO(f, m...) { \ 63 | fprintf(stderr, "[%10s:%4d] " f,__FUNCTION__, __LINE__, ##m); \ 64 | } 65 | 66 | #else 67 | 68 | #define TRACE_INFO(f, m...) (void)0 69 | 70 | #endif /* INFO */ 71 | 72 | #define TRACE_CONFIG(f, m...) fprintf(stderr, f, ##m) 73 | 74 | #ifdef DBGLOG 75 | #define TRACE_LOG(f, m...) TRACE_INFO(f, ##m) 76 | #else 77 | #define TRACE_LOG(f, m...) (void)0 78 | #endif 79 | 80 | #ifdef STREAM 81 | #define TRACE_STREAM(f, m...) TRACE_FUNC("STREAM", f, ##m) 82 | #else 83 | #define TRACE_STREAM(f, m...) (void)0 84 | #endif 85 | 86 | #ifdef STATE 87 | #define TRACE_STATE(f, m...) TRACE_FUNC("STATE", f, ##m) 88 | #else 89 | #define TRACE_STATE(f, m...) (void)0 90 | #endif 91 | 92 | #ifdef SNDBUF 93 | #define TRACE_SNDBUF(f, m...) TRACE_FUNC("SNDBUF", f, ##m) 94 | #else 95 | #define TRACE_SNDBUF(f, m...) (void)0 96 | #endif 97 | 98 | #ifdef RCVBUF 99 | #define TRACE_RCVBUF(f, m...) TRACE_FUNC("RCVBUF", f, ##m) 100 | #else 101 | #define TRACE_RCVBUF(f, m...) (void)0 102 | #endif 103 | 104 | #ifdef CLWND 105 | #define TRACE_CLWND(f, m...) TRACE_FUNC("CLWND", f, ##m) 106 | #else 107 | #define TRACE_CLWND(f, m...) (void)0 108 | #endif 109 | 110 | #ifdef LOSS 111 | #define TRACE_LOSS(f, m...) TRACE_FUNC("LOSS", f, ##m) 112 | #else 113 | #define TRACE_LOSS(f, m...) (void)0 114 | #endif 115 | 116 | #ifdef SACK 117 | #define TRACE_SACK(f, m...) TRACE_FUNC("SACK", f, ##m) 118 | #else 119 | #define TRACE_SACK(f, m...) (void)0 120 | #endif 121 | 122 | #ifdef TSTAMP 123 | #define TRACE_TSTAMP(f, m...) TRACE_FUNC("TSTAMP", f, ##m) 124 | #else 125 | #define TRACE_TSTAMP(f, m...) (void)0 126 | #endif 127 | 128 | #ifdef RTT 129 | #define TRACE_RTT(f, m...) TRACE_FUNC("RTT", f, ##m) 130 | #else 131 | #define TRACE_RTT(f, m...) (void)0 132 | #endif 133 | 134 | #ifdef RTO 135 | #define TRACE_RTO(f, m...) TRACE_FUNC("RTO", f, ##m) 136 | #else 137 | #define TRACE_RTO(f, m...) (void)0 138 | #endif 139 | 140 | #ifdef CONG 141 | #define TRACE_CONG(f, m...) TRACE_FUNC("CONG", f, ##m) 142 | #else 143 | #define TRACE_CONG(f, m...) (void)0 144 | #endif 145 | 146 | #ifdef EPOLL 147 | #define TRACE_EPOLL(f, m...) TRACE_FUNC("EPOLL", f, ##m) 148 | #else 149 | #define TRACE_EPOLL(f, m...) (void)0 150 | #endif 151 | 152 | #ifdef FSTAT 153 | #define TRACE_FSTAT(f, m...) TRACE_FUNC("FSTAT", f, ##m) 154 | #else 155 | #define TRACE_FSTAT(f, m...) (void)0 156 | #endif 157 | 158 | #ifdef APP 159 | #define TRACE_APP(f, m...) TRACE_FUNC("APP", f, ##m) 160 | #else 161 | #define TRACE_APP(f, m...) (void)0 162 | #endif 163 | 164 | #ifdef DBGFIN 165 | #define TRACE_FIN(f, m...) TRACE_FUNC("FIN", f, ##m) 166 | #else 167 | #define TRACE_FIN(f, m...) (void)0 168 | #endif 169 | 170 | #ifdef TSTAT 171 | #define TRACE_TSTAT(f, m...) TRACE_FUNC("TSTAT", f, ##m) 172 | #else 173 | #define TRACE_TSTAT(f, m...) (void)0 174 | #endif 175 | 176 | #ifdef LOOP 177 | #define TRACE_LOOP(f, m...) TRACE_FUNC("LOOP", "ts: %u, "f, cur_ts, ##m) 178 | #else 179 | #define TRACE_LOOP(f, m...) (void)0 180 | #endif 181 | 182 | #ifdef ROUND 183 | #define TRACE_ROUND(f, m...) TRACE_FUNC("ROUND", f, ##m) 184 | #else 185 | #define TRACE_ROUND(f, m...) (void)0 186 | #endif 187 | 188 | #ifdef SELECT 189 | #define TRACE_SELECT(f, m...) TRACE_FUNC("SELECT", f, ##m) 190 | #else 191 | #define TRACE_SELECT(f, m...) (void)0 192 | #endif 193 | 194 | #ifdef API 195 | #define TRACE_API(f, m...) TRACE_FUNC("API", f, ##m) 196 | #else 197 | #define TRACE_API(f, m...) (void)0 198 | #endif 199 | 200 | #ifdef DBGFUNC 201 | 202 | #define TRACE_FUNC(n, f, m...) { \ 203 | thread_printf(mtcp, mtcp->log_fp, "[%6s: %10s:%4d] " \ 204 | f, n, __FUNCTION__, __LINE__, ##m); \ 205 | } 206 | 207 | #else 208 | 209 | #define TRACE_FUNC(f, m...) (void)0 210 | 211 | #endif /* DBGFUNC */ 212 | 213 | void 214 | DumpPacket(mtcp_manager_t mtcp, char *buf, int len, char *step, int ifindex); 215 | 216 | void 217 | DumpIPPacket(mtcp_manager_t mtcp, const struct iphdr *iph, int len); 218 | 219 | void 220 | DumpIPPacketToFile(FILE *fout, const struct iphdr *iph, int len); 221 | 222 | void 223 | flush_log_data(mtcp_manager_t mtcp); 224 | 225 | void 226 | thread_printf(mtcp_manager_t mtcp, FILE* f_idx, const char* _Format, ...); 227 | 228 | #endif /* __DEBUG_H_ */ 229 | --------------------------------------------------------------------------------