├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── deps ├── DEPS └── fetch-deps.sh ├── dp ├── Makefile ├── core │ ├── cfg.c │ ├── control_plane.c │ ├── cpu.c │ ├── dir.mk │ ├── dpdk.c │ ├── echoserver.c │ ├── ethfg.c │ ├── ethqueue.c │ ├── init.c │ ├── kstats.c │ ├── log.c │ ├── mbuf.c │ ├── mem.c │ ├── mempool.c │ ├── nvme_sw_queue.c │ ├── pci.c │ ├── reflex.h │ ├── reflex_ix_client.c │ ├── reflex_server.c │ ├── syscall.c │ ├── tailqueue.c │ ├── timer.c │ └── utimer.c ├── drivers │ ├── dir.mk │ └── nvmedev.c ├── ix.ld ├── lwip │ ├── dir.mk │ ├── inet_chksum.c │ ├── ip4_addr.c │ ├── memp_min.c │ ├── misc.c │ └── pbuf.c └── net │ ├── arp.c │ ├── dir.mk │ ├── dump.c │ ├── icmp.c │ ├── ip.c │ ├── net.c │ ├── net.h │ ├── tcp.c │ ├── tcp_api.c │ ├── tcp_in.c │ ├── tcp_out.c │ └── udp.c ├── inc ├── asm │ ├── chksum.h │ └── cpu.h ├── ix │ ├── atomic.h │ ├── bitmap.h │ ├── byteorder.h │ ├── cfg.h │ ├── compiler.h │ ├── control_plane.h │ ├── cpu.h │ ├── dpdk.h │ ├── errno.h │ ├── ethdev.h │ ├── ethfg.h │ ├── ethqueue.h │ ├── hash.h │ ├── kstats.h │ ├── kstatvectors.h │ ├── list.h │ ├── lock.h │ ├── lock_recursive.h │ ├── log.h │ ├── mbuf.h │ ├── mempool.h │ ├── nvme_sw_queue.h │ ├── nvmedev.h │ ├── pci.h │ ├── profiler.h │ ├── spdk.h │ ├── stddef.h │ ├── syscall.h │ ├── tailqueue.h │ ├── timer.h │ ├── types.h │ └── utimer.h ├── lwip │ ├── arch │ │ ├── cc.h │ │ └── perf.h │ ├── ipv4 │ │ └── lwip │ │ │ ├── icmp.h │ │ │ ├── ip4.h │ │ │ └── ip4_addr.h │ ├── ipv6 │ │ └── lwip │ │ │ ├── ip6.h │ │ │ ├── ip6_addr.h │ │ │ └── nd6.h │ ├── lwip │ │ ├── arch.h │ │ ├── debug.h │ │ ├── def.h │ │ ├── err.h │ │ ├── inet_chksum.h │ │ ├── ip.h │ │ ├── ip_addr.h │ │ ├── mem.h │ │ ├── memp.h │ │ ├── memp_std.h │ │ ├── netif.h │ │ ├── opt.h │ │ ├── pbuf.h │ │ ├── snmp.h │ │ ├── stats.h │ │ ├── sys.h │ │ ├── tcp.h │ │ ├── tcp_impl.h │ │ └── timers.h │ └── lwipopts.h └── net │ ├── arp.h │ ├── ethernet.h │ ├── icmp.h │ ├── ip.h │ └── udp.h ├── ix.conf.sample ├── libix ├── Makefile ├── buf.h ├── ix.h ├── ixev.c ├── ixev.h ├── ixev_timer.c ├── ixev_timer.h ├── libix_syscall.h ├── main.c ├── syscall.h └── syscall_raw.h ├── reflex_nbd ├── Makefile ├── flush_test.c ├── load_eth.sh ├── load_reflex_mod.sh ├── randread_remote.fio ├── reflex_nbd.c ├── reflex_nbd.h └── unload_reflex_mod.sh ├── run_reflex_server.sh ├── sample-aws-ec2.devmodel └── sample.devmodel /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.a 3 | *.d 4 | *.o 5 | *.swp 6 | .depend 7 | 8 | /deps/dpdk 9 | /deps/spdk 10 | /deps/dune 11 | /deps/pcidma 12 | /ix.conf 13 | /dp/ix 14 | /apps/echoclient 15 | /apps/echoserver 16 | /apps/reflex_server 17 | /apps/reflex_ix_client 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2015-2017, Stanford University 4 | 5 | All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are met: 9 | 10 | * Redistributions of source code must retain the above copyright notice, this 11 | list of conditions and the following disclaimer. 12 | 13 | * Redistributions in binary form must reproduce the above copyright notice, 14 | this list of conditions and the following disclaimer in the documentation 15 | and/or other materials provided with the distribution. 16 | 17 | * Neither the name of the copyright holder nor the names of its 18 | contributors may be used to endorse or promote products derived from 19 | this software without specific prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | SUBDIRS = libix dp 23 | CLEANDIRS = $(SUBDIRS:%=clean-%) 24 | 25 | all: $(SUBDIRS) 26 | 27 | apps: libix 28 | 29 | $(SUBDIRS): 30 | $(MAKE) -C $@ 31 | 32 | clean: $(CLEANDIRS) 33 | 34 | style: 35 | astyle -A8 -T8 -p -U -H --suffix=~ -r -Q --exclude=deps --exclude=inc/lwip --exclude=dp/lwip --exclude=dp/net/tcp.c --exclude=dp/net/tcp_in.c --exclude=dp/net/tcp_out.c '*.c' '*.h' 36 | 37 | style-check: 38 | $(eval TEST=$(shell astyle --dry-run --formatted -A8 -T8 -p -U -H --suffix=~ -r -Q --exclude=deps --exclude=inc/lwip --exclude=dp/lwip --exclude=dp/net/tcp.c --exclude=dp/net/tcp_in.c --exclude=dp/net/tcp_out.c '*.c' '*.h' | grep Formatted)) 39 | @if [ -z "$(TEST)" ] ; then\ 40 | echo "success";\ 41 | exit 0;\ 42 | else\ 43 | echo "failed : $(TEST)";\ 44 | exit 1;\ 45 | fi 46 | 47 | $(CLEANDIRS): 48 | $(MAKE) -C $(@:clean-%=%) clean 49 | 50 | .PHONY: all clean style $(SUBDIRS) $(CLEANDIRS) 51 | -------------------------------------------------------------------------------- /deps/DEPS: -------------------------------------------------------------------------------- 1 | # This file is used to manage the dependencies of IX. 2 | 3 | # Three lines of non-changing 4 | # comments so that git can 5 | # automatically merge this file. 6 | git://dpdk.org/dpdk dpdk v17.05 7 | 8 | # Three lines of non-changing 9 | # comments so that git can 10 | # automatically merge this file. 11 | https://github.com/ix-project/pcidma.git pcidma 3a411ad4810a1c8ee7954ad35229dfa9e7f270a1 12 | 13 | # Three lines of non-changing 14 | # comments so that git can 15 | # automatically merge this file. 16 | https://github.com/spdk/spdk.git spdk v17.07 17 | -------------------------------------------------------------------------------- /deps/fetch-deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # fetch-deps 3 | # 4 | # by default, fetch dependencies in DEPS depfile in the current targetdir 5 | # requires git 6 | set -e 7 | 8 | usage () 9 | { 10 | echo "usage: $0 [-h] [-t targetdir] [depfile]" 11 | } 12 | 13 | 14 | init() 15 | { 16 | #optional args 17 | argv__TARGETDIR="" 18 | 19 | getopt_results=$(getopt -s bash -o ht: --long help,targetdir: -- "$@") 20 | if test $? != 0 21 | then 22 | echo "$0: unrecognized option" 23 | exit 1 24 | fi 25 | eval set -- "$getopt_results" 26 | 27 | while true 28 | do 29 | case "$1" in 30 | -h|--help) 31 | usage 32 | exit 0 33 | ;; 34 | -t|--targetdir) 35 | argv__TARGETDIR="$2" 36 | shift 2 37 | ;; 38 | --) 39 | shift 40 | break 41 | ;; 42 | *) 43 | EXCEPTION=$Main__ParameterException 44 | EXCEPTION_MSG="unparseable option $1" 45 | exit "$EXCEPTION" 46 | ;; 47 | esac 48 | done 49 | if [ -n "$argv__TARGETDIR" ]; then 50 | if ! [ -d "$argv__TARGETDIR" ]; then 51 | echo "error" 52 | EXCEPTION=$Main__ParameterException 53 | EXCEPTION_MSG="invalid targetdir parameter $1" 54 | exit "$EXCEPTION" 55 | fi 56 | fi 57 | 58 | # positional args 59 | argv__DEPFILE="" 60 | if [ -n "$1" ]; then 61 | if [ ! -f "$1" ]; then 62 | echo "error" 63 | EXCEPTION=$Main__ParameterException 64 | EXCEPTION_MSG="invalid depfile parameter $1" 65 | exit "$EXCEPTION" 66 | fi 67 | argv__DEPFILE="$1" 68 | else 69 | argv__DEPFILE="$(dirname "$0")/DEPS" 70 | fi 71 | } 72 | 73 | init_exceptions() 74 | { 75 | EXCEPTION=0 76 | EXCEPTION_MSG="" 77 | #Main__Unknown=1 78 | Main__ParameterException=2 79 | } 80 | 81 | cleanup() 82 | { 83 | #clean dir structure in case of script failure 84 | echo "$0: cleanup..." 85 | } 86 | 87 | Main__interruptHandler() 88 | { 89 | # @description signal handler for SIGINT 90 | echo "$0: SIGINT caught" 91 | exit 92 | } 93 | Main__terminationHandler() 94 | { 95 | # @description signal handler for SIGTERM 96 | echo "$0: SIGTERM caught" 97 | exit 98 | } 99 | Main__exitHandler() 100 | { 101 | # @description signal handler for end of the program (clean or unclean). 102 | # probably redundant call, we already call the cleanup in main. 103 | if [ "$EXCEPTION" -ne 0 ] ; then 104 | cleanup 105 | echo "$0: error : ${EXCEPTION_MSG}" 106 | fi 107 | exit 108 | } 109 | 110 | trap Main__interruptHandler INT 111 | trap Main__terminationHandler TERM 112 | trap Main__exitHandler EXIT 113 | 114 | git_clone_checkout() 115 | { 116 | git clone "$1" "$2" || echo "$0: warning: git clone failed." 117 | git -C "$2" checkout "$3" || echo "$0: warning git chechout failed." 118 | } 119 | 120 | Main__main() 121 | { 122 | # init scipt temporals 123 | init_exceptions 124 | init "$@" 125 | #body 126 | if [[ -n "$argv__TARGETDIR" ]]; then 127 | DIR="$argv__TARGETDIR" 128 | else 129 | DIR="$(dirname "$0")" 130 | fi 131 | 132 | while IFS='' read -r line || [[ -n "$line" ]]; do 133 | if [[ $line == \#* || -z $line ]] ; then 134 | continue 135 | fi 136 | local a=( $line ) 137 | local src="${a[0]}" 138 | local dst="${a[1]}" 139 | local tag="${a[2]}" 140 | git_clone_checkout "$src" "$DIR/$dst" "$tag" 141 | done < "$argv__DEPFILE" 142 | 143 | exit 0 144 | } 145 | 146 | # catch signals and exit 147 | #trap exit INT TERM EXIT 148 | 149 | Main__main "$@" 150 | 151 | -------------------------------------------------------------------------------- /dp/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | # A Makefile for IX. 23 | # 24 | # path for this Makefile to work properly. 25 | 26 | DPDK = ../deps/dpdk 27 | SPDK = ../deps/spdk 28 | DPDK_INC = -I$(DPDK)/build/include -I$(DPDK)/lib/librte_eal/common 29 | SPDK_INC = -idirafter$(SPDK)/include 30 | INC = -I../inc -I../inc/lwip -I../inc/lwip/ipv4 -I../inc/lwip/ipv6 $(DPDK_INC) $(SPDK_INC) -I../libix 31 | CC = gcc 32 | 33 | DPDK_MACHINE_FLAGS = -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX -DRTE_MACHINE_CPUFLAG_RDRAND -DRTE_MACHINE_CPUFLAG_FSGSBASE -DRTE_MACHINE_CPUFLAG_F16C -DRTE_MACHINE_CPUFLAG_AVX2 34 | CFLAGS = -g -w -fno-dwarf2-cfi-asm -fno-asynchronous-unwind-tables -O3 -mno-red-zone $(INC) $(EXTRA_CFLAGS) -DSPDK_STRING_H $(DPDK_MACHINE_FLAGS) 35 | PCIDMA = ../deps/pcidma 36 | INC += -I$(PCIDMA) 37 | LD = gcc 38 | LDFLAGS = -T ix.ld 39 | LDLIBS = -lrt -lpthread -lm -lnuma -ldl -lconfig -lpciaccess 40 | 41 | ifneq ($(DEBUG),) 42 | CFLAGS += -DDEBUG 43 | endif 44 | 45 | ifneq ($(ENABLE_KSTATS),) 46 | CFLAGS += -DENABLE_KSTATS 47 | endif 48 | 49 | SRCS = 50 | DIRS = core drivers lwip net 51 | 52 | define register_dir 53 | SRCS += $(patsubst %, $(1)/%, $(2)) 54 | endef 55 | 56 | include $(patsubst %, %/dir.mk, $(DIRS)) 57 | 58 | %.generated.S: %.c 59 | $(CC) $(CFLAGS) -o $@ -S $< 60 | 61 | all: ix 62 | 63 | OBJS=$(subst .c,.o,$(SRCS)) 64 | 65 | depend: .depend 66 | 67 | .depend: $(SRCS) 68 | bash -c "cat $(foreach SRC,$(SRCS),<($(CC) $(CFLAGS) -MM -MT $(SRC:.c=.o) $(SRC))) > ./.depend" 69 | 70 | ifneq ($(MAKECMDGOALS),clean) 71 | -include .depend 72 | endif 73 | 74 | DPDK_LIBS= 75 | DPDK_LIBS+=-Wl,-whole-archive $(DPDK)/build/lib/librte_pmd_ixgbe.a $(DPDK)/build/lib/librte_pmd_ena.a $(DPDK)/build/lib/librte_mempool_ring.a $(DPDK)/build/lib/librte_mempool.a -Wl,-no-whole-archive 76 | DPDK_LIBS+=$(DPDK)/build/lib/librte_mbuf.a 77 | DPDK_LIBS+=$(DPDK)/build/lib/librte_ethdev.a 78 | DPDK_LIBS+=$(DPDK)/build/lib/librte_hash.a 79 | DPDK_LIBS+=$(DPDK)/build/lib/librte_ring.a 80 | DPDK_LIBS+=$(DPDK)/build/lib/librte_eal.a 81 | 82 | SPDK_LIBS=$(SPDK)/build/lib/libspdk_nvme.a 83 | SPDK_LIBS+=$(SPDK)/build/lib/libspdk_util.a 84 | SPDK_LIBS+=$(SPDK)/build/lib/libspdk_env_dpdk.a 85 | SPDK_LIBS+=$(SPDK)/build/lib/libspdk_log.a 86 | 87 | IX_LIBS+=../libix/libix.a 88 | 89 | ix: $(DEPENDENCY) $(OBJS) ix.ld 90 | $(LD) $(LDFLAGS) -o ix $(OBJS) $(SPDK_LIBS) $(DPDK_LIBS) $(LDLIBS) $(IX_LIBS) 91 | clean: 92 | rm -f $(OBJS) ix .depend 93 | 94 | dist-clean: clean 95 | rm *~ 96 | 97 | -------------------------------------------------------------------------------- /dp/core/control_plane.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * control_plane.c - control plane implementation 57 | */ 58 | 59 | #include 60 | 61 | #include 62 | #include 63 | #include 64 | #include 65 | #include 66 | #include 67 | 68 | #include 69 | #include 70 | 71 | volatile struct cp_shmem *cp_shmem; 72 | RTE_DEFINE_PER_LCORE(volatile struct command_struct *, cp_cmd); 73 | 74 | double energy_unit; 75 | 76 | int cp_init(void) 77 | { 78 | int fd, ret; 79 | void *vaddr; 80 | 81 | fd = shm_open("/ix", O_RDWR | O_CREAT | O_TRUNC, 0660); 82 | if (fd == -1) 83 | return 1; 84 | 85 | ret = ftruncate(fd, sizeof(struct cp_shmem)); 86 | if (ret) 87 | return ret; 88 | 89 | vaddr = mmap(NULL, sizeof(struct cp_shmem), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 90 | if (vaddr == MAP_FAILED) 91 | return 1; 92 | 93 | cp_shmem = vaddr; 94 | 95 | bzero((void *)cp_shmem, sizeof(struct cp_shmem)); 96 | cp_shmem->cycles_per_us = cycles_per_us; 97 | 98 | return 0; 99 | } 100 | 101 | void cp_idle(void) 102 | { 103 | int fd, ret; 104 | char buf; 105 | 106 | percpu_get(cp_cmd)->cmd_id = CP_CMD_NOP; 107 | percpu_get(cp_cmd)->status = CP_STATUS_READY; 108 | percpu_get(cp_cmd)->cpu_state = CP_CPU_STATE_IDLE; 109 | fd = open((char *) percpu_get(cp_cmd)->idle.fifo, O_RDONLY); 110 | if (fd != -1) { 111 | ret = read(fd, &buf, 1); 112 | if (ret == -1) 113 | log_err("read on wakeup pipe returned -1 (errno=%d)\n", errno); 114 | close(fd); 115 | } 116 | percpu_get(cp_cmd)->cpu_state = CP_CPU_STATE_RUNNING; 117 | /* NOTE: reset timer position */ 118 | timer_init_cpu(); 119 | } 120 | -------------------------------------------------------------------------------- /dp/core/dir.mk: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | # Makefile for the core system 23 | 24 | SRC = ethfg.c ethqueue.c cfg.c control_plane.c cpu.c init.c log.c mbuf.c mempool.c pci.c utimer.c syscall.c timer.c dpdk.c nvme_sw_queue.c echoserver.c reflex_server.c reflex_ix_client.c 25 | 26 | ifneq ($(ENABLE_KSTATS),) 27 | SRC += kstats.c tailqueue.c 28 | endif 29 | 30 | $(eval $(call register_dir, core, $(SRC))) 31 | 32 | -------------------------------------------------------------------------------- /dp/core/dpdk.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* For memmove and size_t */ 56 | #include 57 | 58 | /* For optind */ 59 | #include 60 | 61 | /* For struct sockaddr */ 62 | #include 63 | 64 | /* Prevent inclusion of rte_memcpy.h */ 65 | //#define _RTE_MEMCPY_X86_64_H_ 66 | //inline void *rte_memcpy(void *dst, const void *src, size_t n); 67 | 68 | /* General DPDK includes */ 69 | #include 70 | #include 71 | #include 72 | #include 73 | #include 74 | #include 75 | 76 | /* IX includes */ 77 | #include 78 | #include 79 | #include 80 | 81 | #include 82 | #include 83 | 84 | struct rte_mempool *dpdk_pool; 85 | 86 | #define MEMPOOL_CACHE_SIZE 256 87 | #define DPDK_MBUF_LENGTH 9000 + RTE_PKTMBUF_HEADROOM 88 | 89 | int dpdk_init(void) 90 | { 91 | struct spdk_env_opts opts; 92 | 93 | spdk_env_opts_init(&opts); 94 | opts.name = "reflex"; 95 | opts.shm_id = -1; 96 | 97 | int core_num = pow(2, cores_active)-1; 98 | char mask[3]; 99 | sprintf(mask, "%x", core_num); 100 | opts.core_mask = mask; 101 | 102 | spdk_env_init(&opts); 103 | 104 | /* pool_size sets an implicit limit on cores * NICs that DPDK allows */ 105 | const int pool_size = 32768; 106 | 107 | //dpdk_pool = rte_pktmbuf_pool_create("mempool", pool_size, MEMPOOL_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 108 | dpdk_pool = rte_pktmbuf_pool_create("mempool", pool_size, MEMPOOL_CACHE_SIZE, 0, DPDK_MBUF_LENGTH, rte_socket_id()); 109 | if (dpdk_pool == NULL) 110 | panic("Cannot create DPDK pool\n"); 111 | 112 | return 0; 113 | } 114 | 115 | -------------------------------------------------------------------------------- /dp/core/log.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * log.c - the logging system 57 | * 58 | * FIXME: Should we direct logs to a file? 59 | */ 60 | 61 | #include 62 | #include 63 | #include 64 | 65 | #include 66 | #include 67 | #include 68 | #include 69 | 70 | #define MAX_LOG_LEN 1024 71 | 72 | __thread bool log_is_early_boot = true; 73 | 74 | int max_loglevel = LOG_DEBUG; 75 | 76 | void logk(int level, const char *fmt, ...) 77 | { 78 | va_list ptr; 79 | char buf[MAX_LOG_LEN]; 80 | time_t ts; 81 | off_t off = 0; 82 | 83 | if (level > max_loglevel) 84 | return; 85 | 86 | if (!log_is_early_boot) { 87 | snprintf(buf, 9, "CPU %02d| ", percpu_get(cpu_id)); 88 | off = strlen(buf); 89 | } 90 | 91 | time(&ts); 92 | off += strftime(buf + off, 32, "%H:%M:%S ", localtime(&ts)); 93 | 94 | snprintf(buf + off, 6, "<%d>: ", level); 95 | off = strlen(buf); 96 | 97 | va_start(ptr, fmt); 98 | vsnprintf(buf + off, MAX_LOG_LEN - off, fmt, ptr); 99 | va_end(ptr); 100 | 101 | printf("%s", buf); 102 | } 103 | 104 | -------------------------------------------------------------------------------- /dp/core/mbuf.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * mbuf.c - buffer management for network packets 57 | * 58 | * TODO: add support for mapping into user-level address space... 59 | */ 60 | 61 | #include 62 | #include 63 | #include 64 | #include 65 | #include 66 | #include 67 | #include 68 | #include 69 | #include 70 | #include 71 | 72 | /* Capacity should be at least RX queues per CPU * ETH_DEV_RX_QUEUE_SZ */ 73 | #define MBUF_CAPACITY 20480 /* Originally set to (768*1024), but decrease # of mbufs allocated and use memory to create larger mbufs for jumbo frames */ 74 | 75 | static struct mempool_datastore mbuf_datastore; 76 | 77 | RTE_DEFINE_PER_LCORE(struct mempool, mbuf_mempool __attribute__((aligned(64)))); 78 | 79 | void mbuf_default_done(struct mbuf *m) 80 | { 81 | mbuf_free(m); 82 | } 83 | 84 | /** 85 | * mbuf_init_cpu - allocates the core-local mbuf region 86 | * 87 | * Returns 0 if successful, otherwise failure. 88 | */ 89 | 90 | int mbuf_init_cpu(void) 91 | { 92 | struct mempool *m = &percpu_get(mbuf_mempool); 93 | return mempool_create(m, &mbuf_datastore, MEMPOOL_SANITY_PERCPU, percpu_get(cpu_id)); 94 | } 95 | 96 | /** 97 | * mbuf_init - allocate global mbuf 98 | */ 99 | 100 | int mbuf_init(void) 101 | { 102 | int ret; 103 | struct mempool_datastore *m = &mbuf_datastore; 104 | ret = mempool_create_mbuf_datastore(m, MBUF_CAPACITY, DPDK_MBUF_SIZE, "mbuf"); 105 | if (ret) { 106 | assert(0); 107 | return ret; 108 | } 109 | //ret = mempool_pagemem_map_to_user(m); 110 | if (ret) { 111 | assert(0); 112 | //mempool_pagemem_destroy(m); 113 | mempool_destroy_datastore(m); 114 | return ret; 115 | } 116 | 117 | return 0; 118 | } 119 | 120 | /** 121 | * mbuf_exit_cpu - frees the core-local mbuf region 122 | */ 123 | void mbuf_exit_cpu(void) 124 | { 125 | mempool_destroy_datastore(&mbuf_datastore); 126 | } 127 | 128 | -------------------------------------------------------------------------------- /dp/core/nvme_sw_queue.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | #include 33 | #include 34 | #include 35 | 36 | #include 37 | #include 38 | #include 39 | #include 40 | 41 | 42 | void nvme_sw_queue_init(struct nvme_sw_queue *q, long fg_handle) 43 | { 44 | q->count = 0; 45 | q->head = 0; 46 | q->tail = 0; 47 | q->total_token_demand = 0; 48 | q->saved_tokens = 0; 49 | q->token_credit = 0; 50 | q->fg_handle = fg_handle; 51 | } 52 | 53 | int nvme_sw_queue_push_back(struct nvme_sw_queue *q, struct nvme_ctx *ctx) 54 | { 55 | if(q->count == NVME_SW_QUEUE_SIZE){ 56 | log_info("nvme_sw_queue full!\n"); 57 | return -EAGAIN; 58 | } 59 | q->buf[q->head] = ctx; 60 | q->head = (q->head + 1) % NVME_SW_QUEUE_SIZE; 61 | q->count++; 62 | q->total_token_demand += ctx->req_cost; 63 | return 0; 64 | } 65 | 66 | int nvme_sw_queue_pop_front(struct nvme_sw_queue *q, struct nvme_ctx **ctx) 67 | { 68 | if(q->count == 0){ 69 | //log_info("ringbuf empty!\n"); 70 | return -EAGAIN; 71 | } 72 | *ctx = q->buf[q->tail]; 73 | q->total_token_demand -= q->buf[q->tail]->req_cost; 74 | q->tail = (q->tail + 1) % NVME_SW_QUEUE_SIZE; 75 | q->count--; 76 | return 0; 77 | } 78 | 79 | int nvme_sw_queue_isempty(struct nvme_sw_queue *q) 80 | { 81 | if (q->count == 0) { 82 | return 1; 83 | } 84 | else { 85 | return 0; 86 | } 87 | } 88 | 89 | int nvme_sw_queue_peak_head_cost(struct nvme_sw_queue *q) 90 | { 91 | if (q->count == 0) 92 | return -1; 93 | 94 | return q->buf[q->tail]->req_cost; 95 | 96 | } 97 | 98 | unsigned long nvme_sw_queue_save_tokens(struct nvme_sw_queue *q, unsigned long tokens) 99 | { 100 | 101 | //only save tokens up to how much have demand, return the rest 102 | if (q->total_token_demand == 0) { 103 | q->saved_tokens = 0; 104 | return 0; 105 | } 106 | if (q->total_token_demand > tokens) { 107 | q->saved_tokens += tokens; 108 | return tokens; 109 | } 110 | if (q->total_token_demand <= tokens) { 111 | q->saved_tokens += q->total_token_demand; 112 | return q->total_token_demand; 113 | } 114 | return 0; 115 | } 116 | 117 | 118 | unsigned long nvme_sw_queue_take_saved_tokens(struct nvme_sw_queue *q) 119 | { 120 | unsigned long saved_tokens = q->saved_tokens; 121 | q->saved_tokens = 0; 122 | return saved_tokens; 123 | } 124 | 125 | -------------------------------------------------------------------------------- /dp/core/reflex.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | 33 | enum msg_type { 34 | PUT, 35 | GET, 36 | PUT_ACK, 37 | GET_RESP, 38 | }; 39 | 40 | struct msg_header { 41 | void* addr; 42 | int cmd; 43 | size_t len; 44 | int tag; 45 | }; 46 | 47 | /* 48 | * ReFlex protocol support 49 | */ 50 | 51 | #define CMD_GET 0x00 52 | #define CMD_SET 0x01 53 | #define CMD_SET_NO_ACK 0x02 54 | 55 | #define RESP_OK 0x00 56 | #define RESP_EINVAL 0x04 57 | 58 | #define REQ_PKT 0x80 59 | #define RESP_PKT 0x81 60 | #define MAX_EXTRA_LEN 8 61 | #define MAX_KEY_LEN 8 62 | 63 | typedef struct __attribute__ ((__packed__)) { 64 | uint16_t magic; 65 | uint16_t opcode; 66 | void *req_handle; 67 | unsigned long lba; 68 | unsigned int lba_count; 69 | } binary_header_blk_t; 70 | 71 | 72 | void *pp_main(void *arg); 73 | 74 | 75 | 76 | 77 | -------------------------------------------------------------------------------- /dp/core/utimer.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | #include 33 | #include 34 | #include 35 | 36 | /* max number of supported user level timers */ 37 | #define UTIMER_COUNT 32 38 | 39 | struct utimer { 40 | struct timer t; 41 | void *cookie; 42 | }; 43 | 44 | struct utimer_list { 45 | struct utimer arr[UTIMER_COUNT]; 46 | }; 47 | 48 | RTE_DEFINE_PER_LCORE(struct utimer_list, utimers); 49 | 50 | void generic_handler(struct timer *t, struct eth_fg *unused) 51 | { 52 | struct utimer *ut; 53 | ut = container_of(t, struct utimer, t); 54 | usys_timer((unsigned long) ut->cookie); 55 | } 56 | 57 | static int find_available(struct utimer_list *tl) 58 | { 59 | static int next; 60 | 61 | if (next >= UTIMER_COUNT) 62 | return -1; 63 | 64 | return next++; 65 | } 66 | 67 | int utimer_init(struct utimer_list *tl, void *udata) 68 | { 69 | struct utimer *ut; 70 | int index; 71 | 72 | index = find_available(tl); 73 | if (index < 0) 74 | return -1; 75 | 76 | ut = &tl->arr[index]; 77 | ut->cookie = udata; 78 | timer_init_entry(&ut->t, generic_handler); 79 | 80 | return index; 81 | } 82 | 83 | int utimer_arm(struct utimer_list *tl, int timer_id, uint64_t delay) 84 | { 85 | struct timer *t; 86 | t = &tl->arr[timer_id].t; 87 | return timer_add(t, NULL, delay); 88 | } 89 | -------------------------------------------------------------------------------- /dp/drivers/dir.mk: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | SRC = nvmedev.c 23 | $(eval $(call register_dir, drivers, $(SRC))) 24 | 25 | -------------------------------------------------------------------------------- /dp/lwip/dir.mk: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | # Temporary makefile for LWIP support code 23 | 24 | SRC = inet_chksum.c ip4_addr.c memp_min.c misc.c pbuf.c 25 | $(eval $(call register_dir, lwip, $(SRC))) 26 | 27 | -------------------------------------------------------------------------------- /dp/lwip/memp_min.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | #include "lwip/memp.h" 56 | #include "lwip/tcp_impl.h" 57 | #include "lwip/timers.h" 58 | 59 | #define MEMP_ALIGN_SIZE(x) (LWIP_MEM_ALIGN_SIZE(x)) 60 | 61 | const u16_t memp_sizes[MEMP_MAX] = { 62 | #define LWIP_MEMPOOL(name,num,size,desc) LWIP_MEM_ALIGN_SIZE(size), 63 | #include 64 | }; 65 | -------------------------------------------------------------------------------- /dp/net/dir.mk: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | # Makefile for network module 23 | 24 | SRC = arp.c dump.c icmp.c ip.c net.c tcp.c tcp_in.c tcp_out.c \ 25 | tcp_api.c #udp.c 26 | $(eval $(call register_dir, net, $(SRC))) 27 | 28 | -------------------------------------------------------------------------------- /dp/net/net.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * net.c - the main file for the network subsystem 57 | */ 58 | 59 | #include 60 | #include 61 | #include 62 | 63 | #include "net.h" 64 | 65 | static void net_dump_cfg(void) 66 | { 67 | char str[IP_ADDR_STR_LEN]; 68 | struct ip_addr mask = {CFG.mask}; 69 | 70 | log_info("net: using the following configuration:\n"); 71 | 72 | ip_addr_to_str((struct ip_addr *)&CFG.host_addr, str); 73 | log_info("\thost IP:\t%s\n", str); 74 | ip_addr_to_str((struct ip_addr *)&CFG.broadcast_addr, str); 75 | log_info("\tbroadcast IP:\t%s\n", str); 76 | ip_addr_to_str((struct ip_addr *)&CFG.gateway_addr, str); 77 | log_info("\tgateway IP:\t%s\n", str); 78 | ip_addr_to_str(&mask, str); 79 | log_info("\tsubnet mask:\t%s\n", str); 80 | } 81 | 82 | /** 83 | * net_init - initializes the network stack 84 | * 85 | * Returns 0 if successful, otherwise fail. 86 | */ 87 | int net_init(void) 88 | { 89 | int ret; 90 | 91 | ret = arp_init(); 92 | if (ret) { 93 | log_err("net: failed to initialize arp\n"); 94 | return ret; 95 | } 96 | 97 | return 0; 98 | } 99 | 100 | /** 101 | * net_cfg - load the network configuration parameters 102 | * 103 | * Returns 0 if successful, otherwise fail. 104 | */ 105 | int net_cfg(void) 106 | { 107 | net_dump_cfg(); 108 | 109 | return 0; 110 | } 111 | 112 | -------------------------------------------------------------------------------- /dp/net/net.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * net.h - the network stack local header 57 | */ 58 | 59 | #pragma once 60 | 61 | #include 62 | #include 63 | #include 64 | 65 | #include 66 | #include 67 | #include 68 | 69 | #include 70 | #include 71 | #include 72 | 73 | /* Address Resolution Protocol (ARP) definitions */ 74 | extern int arp_lookup_mac(struct ip_addr *addr, struct eth_addr *mac); 75 | extern int arp_insert(struct ip_addr *addr, struct eth_addr *mac); 76 | extern void arp_input(struct rte_mbuf *pkt, struct arp_hdr *hdr); 77 | extern int arp_init(void); 78 | 79 | /* Internet Control Message Protocol (ICMP) definitions */ 80 | extern void icmp_input(struct eth_fg *, struct rte_mbuf *pkt, struct icmp_hdr *hdr, int len); 81 | 82 | /* Unreliable Datagram Protocol (UDP) definitions */ 83 | extern void udp_input(struct mbuf *pkt, struct ip_hdr *iphdr, 84 | struct udp_hdr *udphdr); 85 | 86 | /* Transmission Control Protocol (TCP) definitions */ 87 | /* FIXME: change when we integrate better with LWIP */ 88 | extern void tcp_input_tmp(struct eth_fg *, struct rte_mbuf *pkt, struct ip_hdr *iphdr, void *tcphdr); 89 | extern int tcp_api_init(void); 90 | extern int tcp_api_init_fg(void); 91 | 92 | /** 93 | * ip_setup_header - outputs a typical IP header 94 | * @iphdr: a pointer to the header 95 | * @proto: the protocol 96 | * @saddr: the source address 97 | * @daddr: the destination address 98 | * @l4len: the length of the L4 (e.g. UDP or TCP) header and data. 99 | */ 100 | static inline void ip_setup_header(struct ip_hdr *iphdr, uint8_t proto, 101 | uint32_t saddr, uint32_t daddr, 102 | uint16_t l4len) 103 | { 104 | iphdr->header_len = sizeof(struct ip_hdr) / 4; 105 | iphdr->version = 4; 106 | iphdr->tos = 0; 107 | iphdr->len = hton16(sizeof(struct ip_hdr) + l4len); 108 | iphdr->id = 0; 109 | iphdr->off = 0; 110 | iphdr->ttl = 64; 111 | iphdr->proto = proto; 112 | iphdr->chksum = 0; 113 | iphdr->src_addr.addr = hton32(saddr); 114 | iphdr->dst_addr.addr = hton32(daddr); 115 | } 116 | 117 | int ip_send_one(struct eth_fg *cur_fg, struct ip_addr *dst_addr, struct rte_mbuf *pkt, size_t len); 118 | int arp_add_pending_pkt(struct ip_addr *dst_addr, struct eth_fg *fg, struct rte_mbuf *mbuf, size_t len); 119 | -------------------------------------------------------------------------------- /inc/asm/chksum.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * chksum.h - utilities for calculating checksums 57 | */ 58 | 59 | #pragma once 60 | 61 | /** 62 | * chksum_internet - performs an internet checksum on a buffer 63 | * @buf: the buffer 64 | * @len: the length in bytes 65 | * 66 | * An internet checksum is a 16-bit one's complement sum. Details 67 | * are described in RFC 1071. 68 | * 69 | * Returns a 16-bit checksum value. 70 | */ 71 | static inline uint16_t chksum_internet(const char *buf, int len) 72 | { 73 | uint64_t sum; 74 | 75 | asm volatile("xorq %0, %0\n" 76 | 77 | /* process 8 byte chunks */ 78 | "movl %2, %%edx\n" 79 | "shrl $3, %%edx\n" 80 | "cmp $0, %%edx\n" 81 | "jz 2f\n" 82 | "1: adcq (%1), %0\n" 83 | "leaq 8(%1), %1\n" 84 | "decl %%edx\n" 85 | "jne 1b\n" 86 | "adcq $0, %0\n" 87 | 88 | /* process 4 byte (if left) */ 89 | "2: test $4, %2\n" 90 | "je 3f\n" 91 | "movl (%1), %%edx\n" 92 | "addq %%rdx, %0\n" 93 | "adcq $0, %0\n" 94 | "leaq 4(%1), %1\n" 95 | 96 | /* process 2 byte (if left) */ 97 | "3: test $2, %2\n" 98 | "je 4f\n" 99 | "movzxw (%1), %%rdx\n" 100 | "addq %%rdx, %0\n" 101 | "adcq $0, %0\n" 102 | "leaq 2(%1), %1\n" 103 | 104 | /* process 1 byte (if left) */ 105 | "4: test $1, %2\n" 106 | "je 5f\n" 107 | "movzxb (%1), %%rdx\n" 108 | "addq %%rdx, %0\n" 109 | "adcq $0, %0\n" 110 | 111 | /* fold into 16-bit answer */ 112 | "5: movq %0, %1\n" 113 | "shrq $32, %0\n" 114 | "addl %k1, %k0\n" 115 | "adcl $0, %k0\n" 116 | "movq %0, %1\n" 117 | "shrl $16, %k0\n" 118 | "addw %w1, %w0\n" 119 | "adcw $0, %w0\n" 120 | "not %0\n" 121 | 122 | : "=&r"(sum), "=r"(buf) 123 | : "r"(len), "1"(buf) : "%rdx", "cc", "memory"); 124 | 125 | return (uint16_t) sum; 126 | } 127 | 128 | -------------------------------------------------------------------------------- /inc/asm/cpu.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * cpu.h - definitions for x86_64 CPUs 57 | */ 58 | 59 | #pragma once 60 | 61 | /* 62 | * Endianness 63 | */ 64 | 65 | #define __LITTLE_ENDIAN 1234 66 | #define __BIG_ENDIAN 4321 67 | 68 | #define __BYTE_ORDER __LITTLE_ENDIAN 69 | 70 | 71 | /* 72 | * Word Size 73 | */ 74 | 75 | #define __32BIT_WORDS 32 76 | #define __64BIT_WORDS 64 77 | 78 | #define __WORD_SIZE __64BIT_WORDS 79 | 80 | #define CACHE_LINE_SIZE 64 81 | 82 | #define MSR_PKG_ENERGY_STATUS 0x00000611 83 | 84 | #define cpu_relax() asm volatile("pause") 85 | 86 | #define cpu_serialize() \ 87 | asm volatile("cpuid" : : : "%rax", "%rbx", "%rcx", "%rdx") 88 | 89 | static inline unsigned long rdtsc(void) 90 | { 91 | unsigned int a, d; 92 | asm volatile("rdtsc" : "=a"(a), "=d"(d)); 93 | return ((unsigned long) a) | (((unsigned long) d) << 32); 94 | } 95 | 96 | static inline unsigned long rdtscp(unsigned int *aux) 97 | { 98 | unsigned int a, d, c; 99 | asm volatile("rdtscp" : "=a"(a), "=d"(d), "=c"(c)); 100 | if (aux) 101 | *aux = c; 102 | return ((unsigned long) a) | (((unsigned long) d) << 32); 103 | } 104 | 105 | static inline unsigned long rdmsr(unsigned int msr) 106 | { 107 | unsigned low, high; 108 | 109 | asm volatile("rdmsr" : "=a"(low), "=d"(high) : "c"(msr)); 110 | return low | ((unsigned long)high << 32); 111 | } 112 | -------------------------------------------------------------------------------- /inc/ix/bitmap.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * bitmap.h - a library for bit array manipulation 57 | */ 58 | 59 | #pragma once 60 | 61 | #include 62 | 63 | #include 64 | 65 | #define BITS_PER_LONG (sizeof(long) * 8) 66 | #define BITMAP_LONG_SIZE(nbits) \ 67 | div_up(nbits, BITS_PER_LONG) 68 | 69 | #define DEFINE_BITMAP(name, nbits) \ 70 | unsigned long name[BITMAP_LONG_SIZE(nbits)] 71 | 72 | typedef unsigned long *bitmap_ptr; 73 | 74 | #define BITMAP_POS_IDX(pos) ((pos) / BITS_PER_LONG) 75 | #define BITMAP_POS_SHIFT(pos) ((pos) & (BITS_PER_LONG - 1)) 76 | 77 | /** 78 | * bitmap_set - sets a bit in the bitmap 79 | * @bits: the bitmap 80 | * @pos: the bit number 81 | */ 82 | static inline void bitmap_set(unsigned long *bits, int pos) 83 | { 84 | bits[BITMAP_POS_IDX(pos)] |= (1ul << BITMAP_POS_SHIFT(pos)); 85 | } 86 | 87 | /** 88 | * bitmap_clear - clears a bit in the bitmap 89 | * @bits: the bitmap 90 | * @pos: the bit number 91 | */ 92 | static inline void bitmap_clear(unsigned long *bits, int pos) 93 | { 94 | bits[BITMAP_POS_IDX(pos)] &= ~(1ul << BITMAP_POS_SHIFT(pos)); 95 | } 96 | 97 | /** 98 | * bitmap_test - tests if a bit is set in the bitmap 99 | * @bits: the bitmap 100 | * @pos: the bit number 101 | * 102 | * Returns true if the bit is set, otherwise false. 103 | */ 104 | static inline bool bitmap_test(unsigned long *bits, int pos) 105 | { 106 | return (bits[BITMAP_POS_IDX(pos)] & (1ul << BITMAP_POS_SHIFT(pos))) != 0; 107 | } 108 | 109 | /** 110 | * bitmap_init - initializes a bitmap 111 | * @bits: the bitmap 112 | * @nbits: the number of total bits 113 | * @state: if true, all bits are set, otherwise all bits are cleared 114 | */ 115 | static inline void bitmap_init(unsigned long *bits, int nbits, bool state) 116 | { 117 | memset(bits, state ? 0xff : 0x00, BITMAP_LONG_SIZE(nbits) * sizeof(long)); 118 | } 119 | 120 | -------------------------------------------------------------------------------- /inc/ix/byteorder.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * byteorder.h - utilties for swapping bytes and converting endianness 57 | */ 58 | 59 | #pragma once 60 | 61 | #include 62 | #include 63 | 64 | static inline uint16_t __bswap16(uint16_t val) 65 | { 66 | #ifdef HAS_BUILTIN_BSWAP 67 | return __builtin_bswap16(val); 68 | #else 69 | return (((val & 0x00ffU) << 8) | 70 | ((val & 0xff00U) >> 8)); 71 | #endif 72 | } 73 | 74 | static inline uint32_t __bswap32(uint32_t val) 75 | { 76 | #ifdef HAS_BUILTIN_BSWAP 77 | return __builtin_bswap32(val); 78 | #else 79 | return (((val & 0x000000ffUL) << 24) | 80 | ((val & 0x0000ff00UL) << 8) | 81 | ((val & 0x00ff0000UL) >> 8) | 82 | ((val & 0xff000000UL) >> 24)); 83 | #endif 84 | } 85 | 86 | static inline uint64_t __bswap64(uint64_t val) 87 | { 88 | #ifdef HAS_BUILTIN_BSWAP 89 | return __builtin_bswap64(val); 90 | #else 91 | return (((val & 0x00000000000000ffULL) << 56) | 92 | ((val & 0x000000000000ff00ULL) << 40) | 93 | ((val & 0x0000000000ff0000ULL) << 24) | 94 | ((val & 0x00000000ff000000ULL) << 8) | 95 | ((val & 0x000000ff00000000ULL) >> 8) | 96 | ((val & 0x0000ff0000000000ULL) >> 24) | 97 | ((val & 0x00ff000000000000ULL) >> 40) | 98 | ((val & 0xff00000000000000ULL) >> 56)); 99 | #endif 100 | } 101 | 102 | #ifndef __BYTE_ORDER 103 | #error __BYTE_ORDER is undefined 104 | #endif 105 | 106 | #if __BYTE_ORDER == __LITTLE_ENDIAN 107 | 108 | #define cpu_to_le16(x) (x) 109 | #define cpu_to_le32(x) (x) 110 | #define cpu_to_le64(x) (x) 111 | #define cpu_to_be16(x) (__bswap16(x)) 112 | #define cpu_to_be32(x) (__bswap32(x)) 113 | #define cpu_to_be64(x) (__bswap64(x)) 114 | 115 | #define le16_to_cpu(x) (x) 116 | #define le32_to_cpu(x) (x) 117 | #define le64_to_cpu(x) (x) 118 | #define be16_to_cpu(x) (__bswap16(x)) 119 | #define be32_to_cpu(x) (__bswap32(x)) 120 | #define be64_to_cpu(x) (__bswap64(x)) 121 | 122 | #else /* __BYTE_ORDER == __LITLE_ENDIAN */ 123 | 124 | #define cpu_to_le16(x) (__bswap16(x)) 125 | #define cpu_to_le32(x) (__bswap32(x)) 126 | #define cpu_to_le64(x) (__bswap64(x)) 127 | #define cpu_to_be16(x) (x) 128 | #define cpu_to_be32(x) (x) 129 | #define cpu_to_be64(x) (x) 130 | 131 | #define le16_to_cpu(x) (__bswap16(x)) 132 | #define le32_to_cpu(x) (__bswap32(x)) 133 | #define le64_to_cpu(x) (__bswap64(x)) 134 | #define be16_to_cpu(x) (x) 135 | #define be32_to_cpu(x) (x) 136 | #define be64_to_cpu(x) (x) 137 | 138 | #endif /* __BYTE_ORDER == __LITTLE_ENDIAN */ 139 | 140 | #define ntoh16(x) (be16_to_cpu(x)) 141 | #define ntoh32(x) (be32_to_cpu(x)) 142 | #define ntoh64(x) (be64_to_cpu(x)) 143 | 144 | #define hton16(x) (cpu_to_be16(x)) 145 | #define hton32(x) (cpu_to_be32(x)) 146 | #define hton64(x) (cpu_to_be64(x)) 147 | 148 | -------------------------------------------------------------------------------- /inc/ix/cfg.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * cfg.h - configuration parameters 57 | */ 58 | 59 | #pragma once 60 | 61 | #include 62 | #include 63 | 64 | 65 | #define CFG_MAX_PORTS 16 66 | #define CFG_MAX_CPU 128 67 | #define CFG_MAX_ETHDEV 16 68 | #define CFG_MAX_NVMEDEV 1 69 | 70 | enum dev_types { 71 | ETH_DEV, 72 | NVME_DEV, 73 | }; 74 | 75 | enum nvme_dev_models { 76 | DEFAULT_FLASH, // generic device with no token limit 77 | FAKE_FLASH, // no flash: don't schedule nvme requests on device, directly call nvme completion event 78 | FLASH_DEV_MODEL, // flash with request cost model and token limits specified in config input file 79 | }; 80 | 81 | struct cfg_ip_addr { 82 | uint32_t addr; 83 | }; 84 | 85 | struct cfg_parameters { 86 | struct cfg_ip_addr host_addr; 87 | struct cfg_ip_addr broadcast_addr; 88 | struct cfg_ip_addr gateway_addr; 89 | uint32_t mask; 90 | 91 | struct eth_addr mac; 92 | 93 | int num_cpus; 94 | unsigned int cpu[CFG_MAX_CPU]; 95 | 96 | int num_ethdev; 97 | struct pci_addr ethdev[CFG_MAX_ETHDEV]; 98 | 99 | int num_nvmedev; 100 | struct pci_addr nvmedev[CFG_MAX_NVMEDEV]; 101 | 102 | int num_ports; 103 | uint16_t ports[CFG_MAX_PORTS]; 104 | 105 | char loader_path[256]; 106 | }; 107 | 108 | extern struct cfg_parameters CFG; 109 | 110 | int nvme_dev_model; 111 | bool nvme_sched_flag; 112 | 113 | 114 | int NVME_READ_COST; 115 | int NVME_WRITE_COST; 116 | unsigned long MAX_DEV_TOKEN_RATE; 117 | 118 | struct lat_tokenrate_pair{ 119 | uint32_t p95_tail_latency; 120 | uint64_t token_rate_limit; 121 | uint64_t token_rdonly_rate_limit; 122 | }; 123 | 124 | struct lat_tokenrate_pair dev_model[128]; 125 | int dev_model_size; 126 | 127 | extern int cfg_init(int argc, char *argv[], int *args_parsed); 128 | 129 | int cores_active; 130 | -------------------------------------------------------------------------------- /inc/ix/compiler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * compiler.h - useful compiler hints, intrinsics, and attributes 57 | */ 58 | 59 | #pragma once 60 | 61 | #define likely(x) __builtin_expect(!!(x), 1) 62 | #define unlikely(x) __builtin_expect(!!(x), 0) 63 | #define unreachable() __builtin_unreachable() 64 | 65 | #define prefetch0(x) __builtin_prefetch((x), 0, 3) 66 | #define prefetch1(x) __builtin_prefetch((x), 0, 2) 67 | #define prefetch2(x) __builtin_prefetch((x), 0, 1) 68 | #define prefetchnta(x) __builtin_prefetch((x), 0, 0) 69 | #define prefetch() prefetch0() 70 | 71 | #define clz64(x) __builtin_clzll(x) 72 | 73 | #define __packed __attribute__((packed)) 74 | #define __notused __attribute__((unused)) 75 | #define __aligned(x) __attribute__((aligned(x))) 76 | 77 | #define GCC_VERSION (__GNUC__ * 10000 \ 78 | + __GNUC_MINOR__ * 100 \ 79 | + __GNUC_PATCHLEVEL__) 80 | 81 | #if GCC_VERSION >= 40800 82 | #define HAS_BUILTIN_BSWAP 1 83 | #endif 84 | 85 | -------------------------------------------------------------------------------- /inc/ix/control_plane.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * control_plane.h - control plane definitions 57 | */ 58 | 59 | #include 60 | 61 | #include 62 | #include 63 | 64 | #define IDLE_FIFO_SIZE 256 65 | 66 | struct cpu_metrics { 67 | double queuing_delay; 68 | double batch_size; 69 | double queue_size[3]; 70 | long loop_duration; 71 | double idle[3]; 72 | } __aligned(64); 73 | 74 | struct flow_group_metrics { 75 | int cpu; 76 | } __aligned(64); 77 | 78 | enum cpu_state { 79 | CP_CPU_STATE_IDLE = 0, 80 | CP_CPU_STATE_RUNNING, 81 | }; 82 | 83 | enum commands { 84 | CP_CMD_NOP = 0, 85 | CP_CMD_MIGRATE, 86 | CP_CMD_IDLE, 87 | }; 88 | 89 | enum status { 90 | CP_STATUS_READY = 0, 91 | CP_STATUS_RUNNING, 92 | }; 93 | 94 | struct command_struct { 95 | enum cpu_state cpu_state; 96 | enum commands cmd_id; 97 | enum status status; 98 | union { 99 | struct { 100 | DEFINE_BITMAP(fg_bitmap, ETH_MAX_TOTAL_FG); 101 | int cpu; 102 | } migrate; 103 | struct { 104 | char fifo[IDLE_FIFO_SIZE]; 105 | } idle; 106 | }; 107 | char no_idle; 108 | }; 109 | 110 | extern volatile struct cp_shmem { 111 | uint32_t nr_flow_groups; 112 | uint32_t nr_cpus; 113 | float pkg_power; 114 | int cpu[NCPU]; 115 | struct cpu_metrics cpu_metrics[NCPU]; 116 | struct flow_group_metrics flow_group[ETH_MAX_TOTAL_FG]; 117 | struct command_struct command[NCPU]; 118 | uint32_t cycles_per_us; 119 | uint32_t scratchpad_idx; 120 | struct { 121 | long remote_queue_pkts_begin; 122 | long remote_queue_pkts_end; 123 | long local_queue_pkts; 124 | long backlog_before; 125 | long backlog_after; 126 | long timers; 127 | long timer_fired; 128 | long ts_migration_start; 129 | long ts_data_structures_done; 130 | long ts_before_backlog; 131 | long ts_after_backlog; 132 | long ts_migration_end; 133 | long ts_first_pkt_at_prev; 134 | long ts_last_pkt_at_prev; 135 | long ts_first_pkt_at_target; 136 | long ts_last_pkt_at_target; 137 | } scratchpad[1024]; 138 | } *cp_shmem; 139 | 140 | #define SCRATCHPAD (&cp_shmem->scratchpad[cp_shmem->scratchpad_idx]) 141 | #define SCRATCHPAD_NEXT do { assert(++cp_shmem->scratchpad_idx < 1024); } while (0) 142 | 143 | RTE_DECLARE_PER_LCORE(volatile struct command_struct *, cp_cmd); 144 | RTE_DECLARE_PER_LCORE(unsigned long, idle_cycles); 145 | 146 | void cp_idle(void); 147 | 148 | static inline double ema_update(double prv_value, double value, double alpha) 149 | { 150 | return alpha * value + (1 - alpha) * prv_value; 151 | } 152 | 153 | #define EMA_UPDATE(ema, value, alpha) ema = ema_update(ema, value, alpha) 154 | 155 | extern double energy_unit; 156 | -------------------------------------------------------------------------------- /inc/ix/dpdk.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | extern struct rte_mempool *dpdk_pool; 56 | 57 | -------------------------------------------------------------------------------- /inc/ix/kstatvectors.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | 56 | /* 57 | * actually not a pragma once file 58 | */ 59 | 60 | 61 | DEF_KSTATS(none); 62 | DEF_KSTATS(idle); 63 | DEF_KSTATS(user); 64 | DEF_KSTATS(timer); 65 | DEF_KSTATS(timer_collapse); 66 | DEF_KSTATS(print_kstats); 67 | DEF_KSTATS(percpu_bookkeeping); 68 | DEF_KSTATS(tx_reclaim); 69 | DEF_KSTATS(tx_send); 70 | DEF_KSTATS(rx_poll); 71 | DEF_KSTATS(rx_recv); 72 | DEF_KSTATS(bsys); 73 | DEF_KSTATS(timer_tcp_fasttmr); 74 | DEF_KSTATS(timer_tcp_slowtmr); 75 | DEF_KSTATS(eth_input); 76 | DEF_KSTATS(tcp_input_fast_path); 77 | DEF_KSTATS(tcp_input_listen); 78 | DEF_KSTATS(tcp_output_syn); 79 | DEF_KSTATS(tcp_unified_handler); 80 | DEF_KSTATS(timer_tcp_send_delayed_ack); 81 | DEF_KSTATS(timer_handler); 82 | DEF_KSTATS(timer_tcp_retransmit); 83 | DEF_KSTATS(timer_tcp_persist); 84 | DEF_KSTATS(bsys_dispatch_one); 85 | DEF_KSTATS(bsys_tcp_accept); 86 | DEF_KSTATS(bsys_tcp_close); 87 | DEF_KSTATS(bsys_tcp_connect); 88 | DEF_KSTATS(bsys_tcp_recv_done); 89 | DEF_KSTATS(bsys_tcp_reject); 90 | DEF_KSTATS(bsys_tcp_send); 91 | DEF_KSTATS(bsys_tcp_sendv); 92 | DEF_KSTATS(bsys_udp_recv_done); 93 | DEF_KSTATS(bsys_udp_send); 94 | DEF_KSTATS(bsys_udp_sendv); 95 | 96 | DEF_KSTATS(posix_syscall); 97 | -------------------------------------------------------------------------------- /inc/ix/lock.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * lock.h - locking primitives 57 | */ 58 | 59 | #pragma once 60 | 61 | #include 62 | #include 63 | 64 | #define SPINLOCK_INITIALIZER {.locked = 0} 65 | #define DEFINE_SPINLOCK(name) \ 66 | spinlock_t name = SPINLOCK_INITIALIZER 67 | #define DECLARE_SPINLOCK(name) \ 68 | extern spinlock_t name 69 | 70 | /** 71 | * spin_lock_init - prepares a spin lock for use 72 | * @l: the spin lock 73 | */ 74 | static inline void spin_lock_init(spinlock_t *l) 75 | { 76 | l->locked = 0; 77 | } 78 | 79 | /** 80 | * spin_lock - takes a spin lock 81 | * @l: the spin lock 82 | */ 83 | static inline void spin_lock(spinlock_t *l) 84 | { 85 | while (__sync_lock_test_and_set(&l->locked, 1)) { 86 | while (l->locked) { 87 | cpu_relax(); 88 | } 89 | } 90 | } 91 | 92 | /** 93 | * spin_try_lock- takes a spin lock, but only if it is available 94 | * @l: the spin lock 95 | * 96 | * Returns 1 if successful, otherwise 0 97 | */ 98 | static inline bool spin_try_lock(spinlock_t *l) 99 | { 100 | return !(__sync_lock_test_and_set(&l->locked, 1)); 101 | } 102 | 103 | /** 104 | * spin_unlock - releases a spin lock 105 | * @l: the spin lock 106 | */ 107 | static inline void spin_unlock(spinlock_t *l) 108 | { 109 | __sync_lock_release(&l->locked); 110 | } 111 | 112 | -------------------------------------------------------------------------------- /inc/ix/lock_recursive.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * lock.h - locking primitives 34 | */ 35 | 36 | #pragma once 37 | 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | 44 | #define SPINLOCK_INITIALIZER_RECURSIVE {.locked = 0} 45 | #define DEFINE_SPINLOCK_RECURSIVE(name) \ 46 | spinlock_t name = SPINLOCK_INITIALIZER_RECURSIVE 47 | #define DECLARE_SPINLOCK_RECURSIVE(name) \ 48 | extern spinlock_rec_t name 49 | 50 | typedef struct { 51 | volatile unsigned int locked; 52 | volatile unsigned int recursive_count; 53 | } spinlock_rec_t; 54 | 55 | 56 | /** 57 | * spin_lock_recursive_init - prepares a recursive spin lock for use 58 | * @l: the recursive spin lock 59 | */ 60 | static inline void 61 | spin_lock_recursive_init(spinlock_rec_t *l) 62 | { 63 | l->locked = 0; 64 | l->recursive_count = 0; 65 | } 66 | 67 | 68 | /** 69 | * spin_set_recursive - makes this lock a recursive lock 70 | * @l: the spin lock 71 | */ 72 | static inline int 73 | spin_set_recursive(spinlock_rec_t *l) 74 | { 75 | return 0; 76 | } 77 | 78 | static inline void 79 | spin_lock_recursive(spinlock_rec_t *l) 80 | { 81 | unsigned long cid = percpu_get(cpu_id); 82 | 83 | if(l->locked == (long)cid) //we own the lock 84 | l->recursive_count++; 85 | else{ 86 | while(!__sync_bool_compare_and_swap(&l->locked, 0x0UL, cid)) { 87 | while(l->locked) { 88 | cpu_relax(); 89 | } 90 | } 91 | l->recursive_count++; 92 | } 93 | } 94 | 95 | static inline void 96 | spin_unlock_recursive(spinlock_rec_t *l) 97 | { 98 | unsigned long cid = percpu_get(cpu_id); 99 | 100 | if (l->locked != cid) 101 | assert(l->locked == cid); 102 | 103 | l->recursive_count--; 104 | if (l->recursive_count == 0) 105 | __sync_lock_release(&l->locked); 106 | } 107 | -------------------------------------------------------------------------------- /inc/ix/log.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * log.h - the logging service 57 | */ 58 | 59 | #pragma once 60 | 61 | #include 62 | 63 | extern __thread bool log_is_early_boot; 64 | 65 | extern void logk(int level, const char *fmt, ...); 66 | 67 | extern int max_loglevel; 68 | 69 | enum { 70 | LOG_EMERG = 0, /* system is dead */ 71 | LOG_CRIT = 1, /* critical */ 72 | LOG_ERR = 2, /* error */ 73 | LOG_WARN = 3, /* warning */ 74 | LOG_INFO = 4, /* informational */ 75 | LOG_DEBUG = 5, /* debug */ 76 | }; 77 | 78 | #define log_emerg(fmt, ...) logk(LOG_EMERG, fmt, ##__VA_ARGS__) 79 | #define log_crit(fmt, ...) logk(LOG_CRIT, fmt, ##__VA_ARGS__) 80 | #define log_err(fmt, ...) logk(LOG_ERR, fmt, ##__VA_ARGS__) 81 | #define log_warn(fmt, ...) logk(LOG_WARN, fmt, ##__VA_ARGS__) 82 | #define log_info(fmt, ...) logk(LOG_INFO, fmt, ##__VA_ARGS__) 83 | 84 | #ifdef DEBUG 85 | #define log_debug(fmt, ...) logk(LOG_DEBUG, fmt, ##__VA_ARGS__) 86 | #else 87 | #define log_debug(fmt, ...) 88 | #endif 89 | 90 | #define panic(fmt, ...) \ 91 | do {logk(LOG_EMERG, fmt, ##__VA_ARGS__); exit(-1); } while (0) 92 | 93 | -------------------------------------------------------------------------------- /inc/ix/nvme_sw_queue.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Data structure for Flash SW queue scheduling 34 | * Lock-free and works for single producer, single consumer 35 | * 36 | */ 37 | 38 | #include 39 | #include 40 | #define NVME_SW_QUEUE_SIZE (256*8) 41 | 42 | struct nvme_sw_queue 43 | { 44 | struct nvme_ctx* buf[NVME_SW_QUEUE_SIZE]; 45 | int count; // number of elements current in queue 46 | unsigned int head; // head index (insert here) 47 | unsigned int tail; // tail index (remove from here) 48 | unsigned long total_token_demand; 49 | unsigned long saved_tokens; 50 | long fg_handle; 51 | long token_credit; 52 | struct list_node list; 53 | }; 54 | 55 | 56 | 57 | void nvme_sw_queue_init(struct nvme_sw_queue *q, long fg_handle); 58 | int nvme_sw_queue_push_back(struct nvme_sw_queue *q, struct nvme_ctx *ctx); 59 | int nvme_sw_queue_pop_front(struct nvme_sw_queue *q, struct nvme_ctx **ctx); 60 | int nvme_sw_queue_isempty(struct nvme_sw_queue *q); 61 | int nvme_sw_queue_peak_head_cost(struct nvme_sw_queue *q); 62 | unsigned long nvme_sw_queue_save_tokens(struct nvme_sw_queue *q, unsigned long tokens); 63 | unsigned long nvme_sw_queue_take_saved_tokens(struct nvme_sw_queue *q); 64 | 65 | double nvme_sw_queue_save_tokens_fraction(struct nvme_sw_queue *q, double tokens); 66 | double nvme_sw_queue_take_saved_tokens_fraction(struct nvme_sw_queue *q); 67 | -------------------------------------------------------------------------------- /inc/ix/nvmedev.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * nvmedev.h - IX support for nvme interface 34 | */ 35 | 36 | #pragma once 37 | 38 | #include 39 | 40 | #include 41 | #include 42 | #include 43 | 44 | /* FIXME: this should be read from NVMe device register */ 45 | #define MAX_NUM_IO_QUEUES 31 46 | 47 | #define NVME_CMD_READ 0 48 | #define NVME_CMD_WRITE 1 49 | 50 | 51 | #define NVME_MAX_COMPLETIONS 64 52 | 53 | #define MAX_NVME_FLOW_GROUPS 16384 //16 54 | DEFINE_BITMAP(ioq_bitmap, MAX_NUM_IO_QUEUES); 55 | DEFINE_BITMAP(nvme_fgs_bitmap, MAX_NVME_FLOW_GROUPS); 56 | RTE_DECLARE_PER_LCORE(struct spdk_nvme_qpair *, qpair); 57 | 58 | 59 | struct nvme_ctx { 60 | hqu_t handle; 61 | unsigned long cookie; 62 | union user_buf { 63 | void * buf; 64 | struct sgl_buf{ 65 | void **sgl; 66 | int num_sgls; 67 | int current_sgl; 68 | } sgl_buf; 69 | } user_buf; 70 | // added for SW scheduling... 71 | unsigned int tid; //thread id = percpu_get(cpu_nr) 72 | //hqu_t priority; //request priority (determined by flow priority) 73 | hqu_t fg_handle; //flow group handle 74 | int cmd; //NVME_CMD_[READ or WRITE] 75 | int req_cost; //cost of request in tokens 76 | // command arguments... 77 | struct spdk_nvme_ns *ns; //namespace 78 | void* paddr; //physical addr of buffer to write/read to 79 | unsigned long lba; //logical block address 80 | unsigned int lba_count; //size of IO in logical blocks 81 | const struct nvme_completion* completion; //callback function handle 82 | unsigned long time; 83 | }; 84 | 85 | 86 | struct nvme_flow_group { 87 | int flow_group_id; // flow group id (index in bitmap) 88 | //long ns_id; // namespace id 89 | unsigned long cookie; // cookie associated with connection context for user 90 | unsigned int latency_us_SLO; // latency SLO info (0 if best effort) 91 | unsigned long IOPS_SLO; 92 | int rw_ratio_SLO; 93 | unsigned long scaled_IOPS_limit; // calculated based on IOPS, rw_ratio and rw cost 94 | double scaled_IOPuS_limit; 95 | bool latency_critical_flag; 96 | struct nvme_sw_queue* nvme_swq; // thread-local software queue for this flow group 97 | unsigned int tid; // thread id 98 | int conn_ref_count; 99 | }; 100 | 101 | struct nvme_tenant_mgmt { 102 | struct list_head tenant_swq; 103 | int num_tenants; 104 | int num_best_effort_tenants; 105 | }; 106 | 107 | /* 108 | struct nvme_tenant { 109 | long fg_handle; 110 | struct list_node list; 111 | }; 112 | */ 113 | 114 | RTE_DECLARE_PER_LCORE(struct mempool, ctx_mempool); 115 | 116 | RTE_DECLARE_PER_LCORE(int, received_nvme_completions); 117 | 118 | RTE_DECLARE_PER_LCORE(struct nvme_tenant_mgmt, nvme_tenant_manager); 119 | 120 | extern struct nvme_ctx * alloc_local_nvme_ctx(void); 121 | extern void free_local_nvme_ctx(struct nvme_ctx *req); 122 | extern void nvme_process_completions(void); 123 | extern bool nvme_poll_completions(int max_completions); 124 | extern int nvme_schedule(void); 125 | extern int nvme_sched(void); 126 | 127 | -------------------------------------------------------------------------------- /inc/ix/pci.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * pci.h - PCI bus support 57 | */ 58 | 59 | #pragma once 60 | 61 | #include 62 | 63 | struct pci_bar { 64 | uint64_t start; /* the start address, or zero if no resource */ 65 | uint64_t len; /* the length of the resource */ 66 | uint64_t flags; /* Linux resource flags */ 67 | }; 68 | 69 | /* NOTE: these are the same as the Linux PCI sysfs resource flags */ 70 | #define PCI_BAR_IO 0x00000100 71 | #define PCI_BAR_MEM 0x00000200 72 | #define PCI_BAR_PREFETCH 0x00002000 /* typically WC memory */ 73 | #define PCI_BAR_READONLY 0x00004000 /* typically option ROMs */ 74 | 75 | #define SYSFS_PCI_PATH "/sys/bus/pci/devices" 76 | 77 | #define PCI_MAX_BARS 7 78 | 79 | struct pci_addr { 80 | uint16_t domain; 81 | uint8_t bus; 82 | uint8_t slot; 83 | uint8_t func; 84 | }; 85 | 86 | extern int pci_str_to_addr(const char *str, struct pci_addr *addr); 87 | 88 | struct pci_dev { 89 | struct pci_addr addr; 90 | 91 | uint16_t vendor_id; 92 | uint16_t device_id; 93 | uint16_t subsystem_vendor_id; 94 | uint16_t subsystem_device_id; 95 | 96 | struct pci_bar bars[PCI_MAX_BARS]; 97 | int numa_node; 98 | int max_vfs; 99 | }; 100 | 101 | extern struct pci_dev *pci_alloc_dev(const struct pci_addr *addr); 102 | extern struct pci_bar *pci_find_mem_bar(struct pci_dev *dev, int count); 103 | extern void *pci_map_mem_bar(struct pci_dev *dev, struct pci_bar *bar, bool wc); 104 | extern void pci_unmap_mem_bar(struct pci_bar *bar, void *vaddr); 105 | extern int pci_enable_device(struct pci_dev *dev); 106 | extern int pci_set_master(struct pci_dev *dev); 107 | extern int ix_pci_device_cfg_read_u32(struct pci_dev* handle, uint32_t* var, uint32_t offset); 108 | extern int ix_pci_device_cfg_write_u32(struct pci_dev* handle, uint32_t var, uint32_t offset); 109 | -------------------------------------------------------------------------------- /inc/ix/profiler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | #ifdef ENABLE_PROFILER 56 | 57 | void profiler_init(void); 58 | 59 | #else /* ENABLE_PROFILER */ 60 | 61 | static inline void profiler_init(void) { } 62 | 63 | #endif /* ENABLE_PROFILER */ 64 | -------------------------------------------------------------------------------- /inc/ix/tailqueue.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | 56 | #pragma once 57 | 58 | 59 | #define MIN_NINES 1 /* 90% */ 60 | #define MAX_NINES 5 /* 99.999 */ 61 | 62 | const char *tailqueue_nines[] = {"", "90%", "99%", "99.9%", "99.99%", "99.999"}; 63 | 64 | 65 | struct tailqueue; 66 | 67 | struct taildistr { 68 | uint64_t count, min, max; 69 | uint64_t nines[MAX_NINES + 1]; /* nb: nines[0] is unused */ 70 | }; 71 | 72 | extern void tailqueue_addsample(struct tailqueue *tq, 73 | uint64_t t_us); 74 | 75 | extern void tailqueue_calcnines(struct tailqueue *tq, 76 | struct taildistr *td, 77 | int reset); 78 | 79 | 80 | -------------------------------------------------------------------------------- /inc/ix/timer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * timer.h - timer event infrastructure 57 | */ 58 | 59 | #pragma once 60 | 61 | #include 62 | 63 | struct eth_fg; 64 | 65 | struct timer { 66 | struct hlist_node link; 67 | void (*handler)(struct timer *t, struct eth_fg *cur_fg); 68 | uint64_t expires; 69 | int fg_id; 70 | }; 71 | 72 | 73 | #define ONE_SECOND 1000000 74 | #define ONE_MS 1000 75 | #define ONE_US 1 76 | /** 77 | * timer_init_entry - initializes a timer 78 | * @t: the timer 79 | */ 80 | static inline void 81 | timer_init_entry(struct timer *t, void (*handler)(struct timer *t, struct eth_fg *)) 82 | { 83 | t->link.prev = NULL; 84 | t->handler = handler; 85 | } 86 | 87 | /** 88 | * timer_pending - determines if a timer is pending 89 | * @t: the timer 90 | * 91 | * Returns true if the timer is pending, otherwise false. 92 | */ 93 | static inline bool timer_pending(struct timer *t) 94 | { 95 | return t->link.prev != NULL; 96 | } 97 | 98 | extern int timer_add(struct timer *t, struct eth_fg *, uint64_t usecs); 99 | extern void timer_add_for_next_tick(struct timer *t, struct eth_fg *); 100 | extern void timer_add_abs(struct timer *t, struct eth_fg *, uint64_t usecs); 101 | extern uint64_t timer_now(void); 102 | 103 | static inline void __timer_del(struct timer *t) 104 | { 105 | hlist_del(&t->link); 106 | t->link.prev = NULL; 107 | } 108 | 109 | /** 110 | * timer_mod - modifies a timer 111 | * @t: the timer 112 | * @usecs: the number of microseconds from present to fire the timer 113 | * 114 | * If the timer is already armed, then its trigger time is modified. 115 | * Otherwise this function behaves like timer_add(). 116 | * 117 | * Returns 0 if successful, otherwise failure. 118 | */ 119 | static inline int timer_mod(struct timer *t, struct eth_fg *cur_fg, uint64_t usecs) 120 | { 121 | if (timer_pending(t)) 122 | __timer_del(t); 123 | return timer_add(t, cur_fg, usecs); 124 | } 125 | 126 | /** 127 | * timer_del - disarms a timer 128 | * @t: the timer 129 | * 130 | * If the timer is already disarmed, then nothing happens. 131 | */ 132 | static inline void timer_del(struct timer *t) 133 | { 134 | if (timer_pending(t)) 135 | __timer_del(t); 136 | } 137 | 138 | extern void timer_run(void); 139 | extern uint64_t timer_deadline(uint64_t max_us); 140 | 141 | extern int timer_collect_fgs(uint8_t *fg_vector, struct hlist_head *list, uint64_t *timer_pos); 142 | extern void timer_reinject_fgs(struct hlist_head *list, uint64_t timer_pos); 143 | 144 | 145 | extern void timer_init_fg(void); 146 | extern int timer_init_cpu(void); 147 | extern int timer_init(void); 148 | 149 | extern int cycles_per_us; 150 | 151 | int timer_calibrate_tsc(void); 152 | 153 | -------------------------------------------------------------------------------- /inc/ix/types.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * types.h - primitive type definitions 57 | */ 58 | 59 | #pragma once 60 | 61 | #include 62 | #include 63 | 64 | typedef unsigned char uint8_t; 65 | typedef unsigned short uint16_t; 66 | typedef unsigned int uint32_t; 67 | 68 | typedef signed char int8_t; 69 | typedef signed short int16_t; 70 | typedef signed int int32_t; 71 | 72 | #ifndef __WORD_SIZE 73 | #error __WORD_SIZE is undefined 74 | #endif 75 | 76 | #if __WORD_SIZE == __64BIT_WORDS 77 | 78 | typedef unsigned long uint64_t; 79 | typedef signed long int64_t; 80 | 81 | #else /* __WORDSIZE == __64BIT_WORDS */ 82 | 83 | typedef unsigned long long uint64_t; 84 | typedef signed long long int64_t; 85 | 86 | #endif /* __WORDSIZE == __64BIT_WORDS */ 87 | 88 | typedef unsigned long uintptr_t; 89 | typedef long off_t; 90 | typedef unsigned long size_t; 91 | typedef long ssize_t; 92 | 93 | typedef struct { 94 | volatile int locked; 95 | } spinlock_t; 96 | 97 | typedef struct { 98 | int cnt; 99 | } atomic_t; 100 | 101 | typedef struct { 102 | long cnt; 103 | } atomic64_t; 104 | 105 | 106 | typedef struct { 107 | unsigned long cnt; 108 | } atomic_u64_t; 109 | 110 | -------------------------------------------------------------------------------- /inc/ix/utimer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | #pragma once 33 | 34 | 35 | #include 36 | #include 37 | 38 | struct utimer_list; 39 | 40 | RTE_DECLARE_PER_LCORE(struct utimer_list, utimers); 41 | 42 | int utimer_init(struct utimer_list *tl, void *udata); 43 | 44 | int utimer_arm(struct utimer_list *tl, int timer_id, uint64_t delay); 45 | -------------------------------------------------------------------------------- /inc/lwip/arch/cc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * Copyright (c) 2001-2003 Swedish Institute of Computer Science. 26 | * All rights reserved. 27 | * 28 | * Redistribution and use in source and binary forms, with or without modification, 29 | * are permitted provided that the following conditions are met: 30 | * 31 | * 1. Redistributions of source code must retain the above copyright notice, 32 | * this list of conditions and the following disclaimer. 33 | * 2. Redistributions in binary form must reproduce the above copyright notice, 34 | * this list of conditions and the following disclaimer in the documentation 35 | * and/or other materials provided with the distribution. 36 | * 3. The name of the author may not be used to endorse or promote products 37 | * derived from this software without specific prior written permission. 38 | * 39 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 40 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 41 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 42 | * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 43 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 44 | * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 45 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 46 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 47 | * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 48 | * OF SUCH DAMAGE. 49 | * 50 | * This file is part of the lwIP TCP/IP stack. 51 | * 52 | * Author: Adam Dunkels 53 | * 54 | */ 55 | #ifndef __ARCH_CC_H__ 56 | #define __ARCH_CC_H__ 57 | 58 | /* Include some files for defining library routines */ 59 | #include 60 | #include 61 | #include 62 | 63 | #define LWIP_TIMEVAL_PRIVATE 0 64 | 65 | /* Define platform endianness */ 66 | #ifndef BYTE_ORDER 67 | #define BYTE_ORDER LITTLE_ENDIAN 68 | #endif /* BYTE_ORDER */ 69 | 70 | /* Define generic types used in lwIP */ 71 | typedef unsigned char u8_t; 72 | typedef signed char s8_t; 73 | typedef unsigned short u16_t; 74 | typedef signed short s16_t; 75 | typedef unsigned int u32_t; 76 | typedef signed int s32_t; 77 | 78 | typedef unsigned long mem_ptr_t; 79 | 80 | /* Define (sn)printf formatters for these lwIP types */ 81 | #define X8_F "02x" 82 | #define U16_F "hu" 83 | #define S16_F "hd" 84 | #define X16_F "hx" 85 | #define U32_F "u" 86 | #define S32_F "d" 87 | #define X32_F "x" 88 | 89 | /* If only we could use C99 and get %zu */ 90 | #if defined(__x86_64__) 91 | #define SZT_F "lu" 92 | #else 93 | #define SZT_F "u" 94 | #endif 95 | 96 | /* Compiler hints for packing structures */ 97 | #define PACK_STRUCT_FIELD(x) x 98 | #define PACK_STRUCT_STRUCT __attribute__((packed)) 99 | #define PACK_STRUCT_BEGIN 100 | #define PACK_STRUCT_END 101 | 102 | /* prototypes for printf() and abort() */ 103 | #include 104 | #include 105 | /* Plaform specific diagnostic output */ 106 | #define LWIP_PLATFORM_DIAG(x) do {printf x;} while(0) 107 | 108 | #ifdef LWIP_UNIX_EMPTY_ASSERT 109 | #define LWIP_PLATFORM_ASSERT(x) 110 | #else 111 | #define LWIP_PLATFORM_ASSERT(x) do {printf("Assertion \"%s\" failed at line %d in %s\n", \ 112 | x, __LINE__, __FILE__); fflush(NULL); abort();} while(0) 113 | #endif 114 | 115 | #define LWIP_RAND() ((u32_t)rand()) 116 | 117 | #endif /* __ARCH_CC_H__ */ 118 | -------------------------------------------------------------------------------- /inc/lwip/arch/perf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * Copyright (c) 2001-2003 Swedish Institute of Computer Science. 26 | * All rights reserved. 27 | * 28 | * Redistribution and use in source and binary forms, with or without modification, 29 | * are permitted provided that the following conditions are met: 30 | * 31 | * 1. Redistributions of source code must retain the above copyright notice, 32 | * this list of conditions and the following disclaimer. 33 | * 2. Redistributions in binary form must reproduce the above copyright notice, 34 | * this list of conditions and the following disclaimer in the documentation 35 | * and/or other materials provided with the distribution. 36 | * 3. The name of the author may not be used to endorse or promote products 37 | * derived from this software without specific prior written permission. 38 | * 39 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 40 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 41 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 42 | * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 43 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 44 | * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 45 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 46 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 47 | * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 48 | * OF SUCH DAMAGE. 49 | * 50 | * This file is part of the lwIP TCP/IP stack. 51 | * 52 | * Author: Adam Dunkels 53 | * 54 | */ 55 | #ifndef __ARCH_PERF_H__ 56 | #define __ARCH_PERF_H__ 57 | 58 | #include 59 | 60 | #ifdef PERF 61 | #define PERF_START { \ 62 | unsigned long __c1l, __c1h, __c2l, __c2h; \ 63 | __asm__(".byte 0x0f, 0x31" : "=a" (__c1l), "=d" (__c1h)) 64 | #define PERF_STOP(x) __asm__(".byte 0x0f, 0x31" : "=a" (__c2l), "=d" (__c2h)); \ 65 | perf_print(__c1l, __c1h, __c2l, __c2h, x);} 66 | 67 | /*#define PERF_START do { \ 68 | struct tms __perf_start, __perf_end; \ 69 | times(&__perf_start) 70 | #define PERF_STOP(x) times(&__perf_end); \ 71 | perf_print_times(&__perf_start, &__perf_end, x);\ 72 | } while(0)*/ 73 | #else /* PERF */ 74 | #define PERF_START /* null definition */ 75 | #define PERF_STOP(x) /* null definition */ 76 | #endif /* PERF */ 77 | 78 | void perf_print(unsigned long c1l, unsigned long c1h, 79 | unsigned long c2l, unsigned long c2h, 80 | char *key); 81 | 82 | void perf_print_times(struct tms *start, struct tms *end, char *key); 83 | 84 | void perf_init(char *fname); 85 | 86 | #endif /* __ARCH_PERF_H__ */ 87 | -------------------------------------------------------------------------------- /inc/lwip/lwip/err.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * Copyright (c) 2001-2004 Swedish Institute of Computer Science. 26 | * All rights reserved. 27 | * 28 | * Redistribution and use in source and binary forms, with or without modification, 29 | * are permitted provided that the following conditions are met: 30 | * 31 | * 1. Redistributions of source code must retain the above copyright notice, 32 | * this list of conditions and the following disclaimer. 33 | * 2. Redistributions in binary form must reproduce the above copyright notice, 34 | * this list of conditions and the following disclaimer in the documentation 35 | * and/or other materials provided with the distribution. 36 | * 3. The name of the author may not be used to endorse or promote products 37 | * derived from this software without specific prior written permission. 38 | * 39 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 40 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 41 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 42 | * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 43 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 44 | * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 45 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 46 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 47 | * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 48 | * OF SUCH DAMAGE. 49 | * 50 | * This file is part of the lwIP TCP/IP stack. 51 | * 52 | * Author: Adam Dunkels 53 | * 54 | */ 55 | #ifndef __LWIP_ERR_H__ 56 | #define __LWIP_ERR_H__ 57 | 58 | #include "lwip/opt.h" 59 | #include "lwip/arch.h" 60 | 61 | #ifdef __cplusplus 62 | extern "C" { 63 | #endif 64 | 65 | /** Define LWIP_ERR_T in cc.h if you want to use 66 | * a different type for your platform (must be signed). */ 67 | #ifdef LWIP_ERR_T 68 | typedef LWIP_ERR_T err_t; 69 | #else /* LWIP_ERR_T */ 70 | typedef s8_t err_t; 71 | #endif /* LWIP_ERR_T*/ 72 | 73 | /* Definitions for error constants. */ 74 | 75 | #define ERR_OK 0 /* No error, everything OK. */ 76 | #define ERR_MEM -1 /* Out of memory error. */ 77 | #define ERR_BUF -2 /* Buffer error. */ 78 | #define ERR_TIMEOUT -3 /* Timeout. */ 79 | #define ERR_RTE -4 /* Routing problem. */ 80 | #define ERR_INPROGRESS -5 /* Operation in progress */ 81 | #define ERR_VAL -6 /* Illegal value. */ 82 | #define ERR_WOULDBLOCK -7 /* Operation would block. */ 83 | #define ERR_USE -8 /* Address in use. */ 84 | #define ERR_ISCONN -9 /* Already connected. */ 85 | 86 | #define ERR_IS_FATAL(e) ((e) < ERR_ISCONN) 87 | 88 | #define ERR_ABRT -10 /* Connection aborted. */ 89 | #define ERR_RST -11 /* Connection reset. */ 90 | #define ERR_CLSD -12 /* Connection closed. */ 91 | #define ERR_CONN -13 /* Not connected. */ 92 | 93 | #define ERR_ARG -14 /* Illegal argument. */ 94 | 95 | #define ERR_IF -15 /* Low-level netif error */ 96 | 97 | 98 | #ifdef LWIP_DEBUG 99 | extern const char *lwip_strerr(err_t err); 100 | #else 101 | #define lwip_strerr(x) "" 102 | #endif /* LWIP_DEBUG */ 103 | 104 | #ifdef __cplusplus 105 | } 106 | #endif 107 | 108 | #endif /* __LWIP_ERR_H__ */ 109 | -------------------------------------------------------------------------------- /inc/lwip/lwip/mem.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | #ifndef __LWIP_MEM_H__ 25 | #define __LWIP_MEM_H__ 26 | 27 | typedef size_t mem_size_t; 28 | 29 | /** mem_init is not used when using pools instead of a heap */ 30 | #define mem_init() 31 | /** mem_trim is not used when using pools instead of a heap: 32 | we can't free part of a pool element and don't want to copy the rest */ 33 | #define mem_trim(mem, size) (mem) 34 | void *mem_malloc(mem_size_t size); 35 | void mem_free(void *mem); 36 | 37 | /** Calculate memory size for an aligned buffer - returns the next highest 38 | * multiple of MEM_ALIGNMENT (e.g. LWIP_MEM_ALIGN_SIZE(3) and 39 | * LWIP_MEM_ALIGN_SIZE(4) will both yield 4 for MEM_ALIGNMENT == 4). 40 | */ 41 | #ifndef LWIP_MEM_ALIGN_SIZE 42 | #define LWIP_MEM_ALIGN_SIZE(size) (((size) + MEM_ALIGNMENT - 1) & ~(MEM_ALIGNMENT-1)) 43 | #endif 44 | 45 | /** Align a memory pointer to the alignment defined by MEM_ALIGNMENT 46 | * so that ADDR % MEM_ALIGNMENT == 0 47 | */ 48 | #ifndef LWIP_MEM_ALIGN 49 | #define LWIP_MEM_ALIGN(addr) ((void *)(((mem_ptr_t)(addr) + MEM_ALIGNMENT - 1) & ~(mem_ptr_t)(MEM_ALIGNMENT-1))) 50 | #endif 51 | 52 | #endif /* __LWIP_MEM_H__ */ 53 | -------------------------------------------------------------------------------- /inc/lwip/lwip/memp.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | #ifndef __LWIP_MEMP_H__ 25 | #define __LWIP_MEMP_H__ 26 | 27 | #include 28 | 29 | #include 30 | #include 31 | #include 32 | 33 | #include "lwip/opt.h" 34 | 35 | /* Create the list of all memory pools managed by memp. MEMP_MAX represents a NULL pool at the end */ 36 | typedef enum { 37 | #define LWIP_MEMPOOL(name,num,size,desc) MEMP_##name, 38 | #include "lwip/memp_std.h" 39 | MEMP_MAX 40 | } memp_t; 41 | 42 | extern const u16_t memp_sizes[MEMP_MAX]; 43 | 44 | int memp_init(void); 45 | int memp_init_cpu(void); 46 | 47 | RTE_DECLARE_PER_LCORE(struct mempool, pbuf_mempool); 48 | RTE_DECLARE_PER_LCORE(struct mempool, pbuf_with_payload_mempool); 49 | RTE_DECLARE_PER_LCORE(struct mempool, tcp_pcb_mempool); 50 | RTE_DECLARE_PER_LCORE(struct mempool, tcp_pcb_listen_mempool); 51 | RTE_DECLARE_PER_LCORE(struct mempool, tcp_seg_mempool); 52 | 53 | static inline void *memp_malloc(memp_t type) 54 | { 55 | switch (type) { 56 | case MEMP_PBUF: 57 | return mempool_alloc(&percpu_get(pbuf_mempool)); 58 | case MEMP_TCP_PCB: 59 | return mempool_alloc(&percpu_get(tcp_pcb_mempool)); 60 | case MEMP_TCP_PCB_LISTEN: 61 | return mempool_alloc(&percpu_get(tcp_pcb_listen_mempool)); 62 | case MEMP_TCP_SEG: 63 | return mempool_alloc(&percpu_get(tcp_seg_mempool)); 64 | case MEMP_SYS_TIMEOUT: 65 | case MEMP_PBUF_POOL: 66 | case MEMP_MAX: 67 | break; 68 | } 69 | 70 | return NULL; 71 | } 72 | 73 | static inline void memp_free(memp_t type, void *mem) 74 | { 75 | switch (type) { 76 | case MEMP_PBUF: 77 | mempool_free(&percpu_get(pbuf_mempool), mem); 78 | return; 79 | case MEMP_TCP_PCB: 80 | mempool_free(&percpu_get(tcp_pcb_mempool), mem); 81 | return; 82 | case MEMP_TCP_PCB_LISTEN: 83 | mempool_free(&percpu_get(tcp_pcb_listen_mempool), mem); 84 | return; 85 | case MEMP_TCP_SEG: 86 | mempool_free(&percpu_get(tcp_seg_mempool), mem); 87 | return; 88 | case MEMP_SYS_TIMEOUT: 89 | case MEMP_PBUF_POOL: 90 | case MEMP_MAX: 91 | break; 92 | } 93 | } 94 | 95 | #endif /* __LWIP_MEMP_H__ */ 96 | -------------------------------------------------------------------------------- /inc/lwip/lwip/timers.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * Copyright (c) 2001-2004 Swedish Institute of Computer Science. 26 | * All rights reserved. 27 | * 28 | * Redistribution and use in source and binary forms, with or without modification, 29 | * are permitted provided that the following conditions are met: 30 | * 31 | * 1. Redistributions of source code must retain the above copyright notice, 32 | * this list of conditions and the following disclaimer. 33 | * 2. Redistributions in binary form must reproduce the above copyright notice, 34 | * this list of conditions and the following disclaimer in the documentation 35 | * and/or other materials provided with the distribution. 36 | * 3. The name of the author may not be used to endorse or promote products 37 | * derived from this software without specific prior written permission. 38 | * 39 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 40 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 41 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 42 | * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 43 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 44 | * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 45 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 46 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 47 | * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 48 | * OF SUCH DAMAGE. 49 | * 50 | * This file is part of the lwIP TCP/IP stack. 51 | * 52 | * Author: Adam Dunkels 53 | * Simon Goldschmidt 54 | * 55 | */ 56 | #ifndef __LWIP_TIMERS_H__ 57 | #define __LWIP_TIMERS_H__ 58 | 59 | #include "lwip/opt.h" 60 | 61 | /* Timers are not supported when NO_SYS==1 and NO_SYS_NO_TIMERS==1 */ 62 | #define LWIP_TIMERS (!NO_SYS || (NO_SYS && !NO_SYS_NO_TIMERS)) 63 | 64 | #if LWIP_TIMERS 65 | 66 | #include "lwip/err.h" 67 | #if !NO_SYS 68 | #include "lwip/sys.h" 69 | #endif 70 | 71 | #ifdef __cplusplus 72 | extern "C" { 73 | #endif 74 | 75 | #ifndef LWIP_DEBUG_TIMERNAMES 76 | #ifdef LWIP_DEBUG 77 | #define LWIP_DEBUG_TIMERNAMES SYS_DEBUG 78 | #else /* LWIP_DEBUG */ 79 | #define LWIP_DEBUG_TIMERNAMES 0 80 | #endif /* LWIP_DEBUG*/ 81 | #endif 82 | 83 | /** Function prototype for a timeout callback function. Register such a function 84 | * using sys_timeout(). 85 | * 86 | * @param arg Additional argument to pass to the function - set up by sys_timeout() 87 | */ 88 | typedef void (* sys_timeout_handler)(void *arg); 89 | 90 | struct sys_timeo { 91 | struct sys_timeo *next; 92 | u32_t time; 93 | sys_timeout_handler h; 94 | void *arg; 95 | #if LWIP_DEBUG_TIMERNAMES 96 | const char* handler_name; 97 | #endif /* LWIP_DEBUG_TIMERNAMES */ 98 | }; 99 | 100 | void sys_timeouts_init(void); 101 | 102 | #if LWIP_DEBUG_TIMERNAMES 103 | void sys_timeout_debug(u32_t msecs, sys_timeout_handler handler, void *arg, const char* handler_name); 104 | #define sys_timeout(msecs, handler, arg) sys_timeout_debug(msecs, handler, arg, #handler) 105 | #else /* LWIP_DEBUG_TIMERNAMES */ 106 | void sys_timeout(u32_t msecs, sys_timeout_handler handler, void *arg); 107 | #endif /* LWIP_DEBUG_TIMERNAMES */ 108 | 109 | void sys_untimeout(sys_timeout_handler handler, void *arg); 110 | #if NO_SYS 111 | void sys_check_timeouts(void); 112 | void sys_restart_timeouts(void); 113 | #else /* NO_SYS */ 114 | void sys_timeouts_mbox_fetch(sys_mbox_t *mbox, void **msg); 115 | #endif /* NO_SYS */ 116 | 117 | 118 | #ifdef __cplusplus 119 | } 120 | #endif 121 | 122 | #endif /* LWIP_TIMERS */ 123 | #endif /* __LWIP_TIMERS_H__ */ 124 | -------------------------------------------------------------------------------- /inc/lwip/lwipopts.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | #define LWIP_STATS 0 56 | #define LWIP_TCP 1 57 | #define NO_SYS 1 58 | #define LWIP_RAW 0 59 | #define LWIP_UDP 0 60 | #define IP_REASSEMBLY 0 61 | #define IP_FRAG 0 62 | #define LWIP_NETCONN 0 63 | 64 | #define MEM_LIBC_MALLOC 1 65 | #define MEMP_MEM_MALLOC 1 66 | 67 | //#define LWIP_DEBUG LWIP_DBG_OFF 68 | #undef LWIP_DEBUG 69 | #define TCP_CWND_DEBUG LWIP_DBG_OFF 70 | #define TCP_DEBUG LWIP_DBG_OFF 71 | #define TCP_FR_DEBUG LWIP_DBG_OFF 72 | #define TCP_INPUT_DEBUG LWIP_DBG_OFF 73 | #define TCP_OUTPUT_DEBUG LWIP_DBG_OFF 74 | #define TCP_QLEN_DEBUG LWIP_DBG_OFF 75 | #define TCP_RST_DEBUG LWIP_DBG_OFF 76 | #define TCP_RTO_DEBUG LWIP_DBG_OFF 77 | #define TCP_WND_DEBUG LWIP_DBG_OFF 78 | 79 | #include 80 | #include 81 | 82 | #define LWIP_IX 83 | 84 | #define LWIP_PLATFORM_BYTESWAP 1 85 | #define LWIP_PLATFORM_HTONS(x) hton16(x) 86 | #define LWIP_PLATFORM_NTOHS(x) ntoh16(x) 87 | #define LWIP_PLATFORM_HTONL(x) hton32(x) 88 | #define LWIP_PLATFORM_NTOHL(x) ntoh32(x) 89 | 90 | #define LWIP_WND_SCALE 1 91 | #define TCP_RCV_SCALE 7 92 | #define TCP_SND_BUF 65536 93 | 94 | /* 95 | * FIXME: TCP_MSS of 8960 causes traffic to get dropped in AWS 96 | * perhaps jumbo frames is not supported 97 | * so conservatively set TCP_MSS to 1460 for now 98 | */ 99 | //#define TCP_MSS 8960 /* Originally 1460, but now support jumbo frames */ 100 | //#define TCP_MSS 1460 101 | #define TCP_MSS 6000 //TODO: increase this to 9000 after debug AWS issue 102 | 103 | //#define TCP_WND (1024 * TCP_MSS) //Not sure what correct TCP_WND setting should be 104 | //#define TCP_WND (2048 * 1460) //Not sure what correct TCP_WND setting should be 105 | #define TCP_WND 1<<15 106 | 107 | #define CHECKSUM_CHECK_IP 0 108 | #define CHECKSUM_CHECK_TCP 0 109 | #define TCP_ACK_DELAY (1 * ONE_MS) 110 | #define RTO_UNITS (500 * ONE_MS) 111 | 112 | /* EdB 2014-11-07 */ 113 | #define LWIP_NOASSERT 114 | #define LWIP_EVENT_API 1 115 | #define LWIP_NETIF_HWADDRHINT 1 116 | 117 | -------------------------------------------------------------------------------- /inc/net/arp.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * arp.h - Address Resolution Protocol (RFC 826, RFC 903) 57 | */ 58 | 59 | #pragma once 60 | 61 | #include 62 | #include 63 | 64 | struct arp_hdr { 65 | uint16_t htype; 66 | uint16_t ptype; /* the ETHERTYPE */ 67 | uint8_t hlen; 68 | uint8_t plen; 69 | uint16_t op; 70 | 71 | /* 72 | * Variable length fields continue as follows: 73 | * sender hw addr: hlen bytes 74 | * sender protocol addr: plen bytes 75 | * target hw addr: hlen bytes 76 | * target protocol addr: plen bytes 77 | */ 78 | } __packed; 79 | 80 | struct arp_hdr_ethip { 81 | struct eth_addr sender_mac; 82 | struct ip_addr sender_ip; 83 | struct eth_addr target_mac; 84 | struct ip_addr target_ip; 85 | } __packed; 86 | 87 | #define ARP_HTYPE_ETHER 1 /* ethernet */ 88 | #define ARP_HTYPE_IEEE802 6 /* token-ring */ 89 | #define ARP_HTYPE_ARCNET 7 /* arcnet */ 90 | #define ARP_HTYPE_FRELAY 16 /* frame relay */ 91 | #define ARP_HTYPE_IEEE1394 24 /* firewire */ 92 | #define ARP_HTYPE_INFINIBAND 32 /* infiniband */ 93 | 94 | enum { 95 | ARP_OP_REQUEST = 1, /* request hw addr given protocol addr */ 96 | ARP_OP_REPLY = 2, /* response hw addr given protocol addr */ 97 | ARP_OP_REVREQUEST = 3, /* request protocol addr given hw addr */ 98 | ARP_OP_REVREPLY = 4, /* response protocol addr given hw addr */ 99 | }; 100 | 101 | -------------------------------------------------------------------------------- /inc/net/udp.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * udp.h - Unreliable Datagram Protocol 57 | */ 58 | 59 | #pragma once 60 | 61 | #include 62 | 63 | struct udp_hdr { 64 | uint16_t src_port; 65 | uint16_t dst_port; 66 | uint16_t len; 67 | uint16_t chksum; 68 | }; 69 | 70 | -------------------------------------------------------------------------------- /ix.conf.sample: -------------------------------------------------------------------------------- 1 | # ix.conf 2 | # configuration file for IX 3 | # 4 | # This file defines the configuration for IX data center OS. You should 5 | # carefully update all fields in accordence with your local configuration 6 | # before running IX for the first time. 7 | 8 | ############################################################################### 9 | # Network Parameters 10 | ############################################################################### 11 | 12 | ## host_addr : CIDR IP address and netmask that will be assigned to 13 | ## the adapter once it comes up. 14 | host_addr="192.168.40.1/16" 15 | 16 | # gateway_addr : default gateway IP address 17 | gateway_addr="192.168.1.1" 18 | 19 | ## port : port(s) that will be bound to the application by the 20 | ## kernel when launching IX. 21 | ## You can specify multiple entries, e.g. 'port=[X, Y, Z]' 22 | port=1234 23 | 24 | ############################################################################### 25 | # Hardware parameters 26 | ############################################################################### 27 | 28 | ## devices : Specifies the PCI device ID of the adapter to bind IX with. 29 | ## Should be an Intel compatible NIC (e.g. 82599, X520, X540, etc).. 30 | ## See the Docs to see which adapters are currently supported. 31 | ## Format is a list dddd:bb:ss.ff,... d - domain, b = bus, 32 | ## s = slot, f = function. Usually, `lspci | grep Ethernet` allows to see 33 | ## available Ethernet controllers. Similarly, `lspci | grep Non-Volatile` 34 | ## allows to see available NVMe SSD controllers. 35 | ## You can specify multiple entries, e.g. 'devices=["X","Y","Z"]' 36 | devices="0:05:00.0" 37 | nvme_devices="0:04:00.0" 38 | 39 | ############################################################################### 40 | # ReFlex I/O scheduler parameters 41 | ############################################################################### 42 | # nvme_device_model: "sample.devmodel" is a sample config file for a device model 43 | # which specifies read/write costs and token limits for 44 | # tail latency SLOs; values are device-specific 45 | # You should follow instructions in this file to modify 46 | # values corresponding to the performance of your SSD. 47 | # Either modify sample.devmodel directly or copy to a 48 | # different file and specify file name here. 49 | # "default" for no token limits 50 | # "fake" models ultra low latency device since we don't 51 | # submit I/Os to real device, just generate fake I/O 52 | # completion events (can be useful for perf debugging) 53 | # 54 | # scheduler: "on" (by default) 55 | # "off" means I/O submitted directly to flash, 56 | # no SW queueing, no QoS scheduling 57 | nvme_device_model="sample.devmodel" 58 | scheduler="on" 59 | 60 | ## cpu : Indicates which CPU process unit(s) (P) this IX instance 61 | ## should be bound to. 62 | ## WARNING: If this setting defines multiple nodes, make sure they ar 63 | ## part of the same NUMA socket. On most architectures, the IDs of 64 | ## processing units on the same socket are interleaved. 65 | ## Usually `lstopo` allows to see NUMA nodes topology 66 | ## You can specify multiple entries, e.g. 'nodes=["X","Y","Z"]' 67 | cpu=0 68 | 69 | ## batch : Specifies maximum batch size of received packets to process. 70 | ## Default: 64. 71 | batch=64 72 | 73 | ## loader_path : kernel loader to use with IX module: 74 | ## 75 | loader_path="/lib64/ld-linux-x86-64.so.2" 76 | 77 | ############################################################################### 78 | # Optional parameters 79 | ############################################################################### 80 | 81 | ## arp: Allows you to manually add static arp entries in the interface arp table 82 | # 83 | #arp=( 84 | # { 85 | # ip : "192.168.1.2" 86 | # mac : "aa:aa:aa:aa:aa:aa" 87 | # }, 88 | # { 89 | # ip : "192.168.1.3" 90 | # mac : "bb:bb:bb:bb:bb:bb" 91 | # } 92 | #) 93 | 94 | 95 | ## fdir : Static flow director rules to be added to NIC 96 | # Use this to steer traffic to particular cores when multi-core ReFlex. 97 | # Queue number corresponds to CPU core id 98 | # 99 | ##fdir=( 100 | # { 101 | # dst_ip : "192.168.40.1" 102 | # src_ip : "XX.XX.XX.XX" 103 | # dst_port : 1234 104 | # queue : 0 105 | # }, 106 | # { 107 | # dst_ip : "192.168.40.1" 108 | # src_ip : "YY.YY.YY.YY" 109 | # dst_port : 5678 110 | # queue : 1 111 | # } 112 | #) 113 | 114 | -------------------------------------------------------------------------------- /libix/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | # A Makefile for libIX. 23 | 24 | DPDK = ../deps/dpdk 25 | DPDK_INC = -I$(DPDK)/build/include -I$(DPDK)/lib/librte_eal/common -I$(DPDK)/drivers/net/ixgbe 26 | DPDK_LIBS= 27 | #DPDK_LIBS+=-Wl,-whole-archive $(DPDK)/build/lib/librte_pmd_ixgbe.a -Wl,-no-whole-archive 28 | DPDK_LIBS+=$(DPDK)/build/lib/librte_mbuf.a 29 | DPDK_LIBS+=$(DPDK)/build/lib/librte_ethdev.a 30 | DPDK_LIBS+=$(DPDK)/build/lib/librte_mempool.a 31 | DPDK_LIBS+=$(DPDK)/build/lib/librte_ring.a 32 | DPDK_LIBS+=$(DPDK)/build/lib/librte_eal.a 33 | 34 | DPDK_MACHINE_FLAGS = -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX -DRTE_MACHINE_CPUFLAG_RDRAND -DRTE_MACHINE_CPUFLAG_FSGSBASE -DRTE_MACHINE_CPUFLAG_F16C -DRTE_MACHINE_CPUFLAG_AVX2 35 | 36 | INC = -I. -I../inc $(DPDK_INC) 37 | CC = gcc 38 | CFLAGS = -g -Wall -O3 $(INC) $(DPDK_MACHINE_FLAGS) 39 | AR = ar 40 | 41 | SRCS = main.c ixev.c ixev_timer.c 42 | OBJS = $(subst .c,.o,$(SRCS)) 43 | 44 | all: libix.a 45 | 46 | depend: .depend 47 | 48 | .depend: $(SRCS) 49 | rm -f ./.depend 50 | $(foreach SRC,$(SRCS),$(CC) $(CFLAGS) -MM -MT $(SRC:.c=.o) $(SRC) >> .depend;) 51 | 52 | -include .depend 53 | 54 | libix.a: $(OBJS) 55 | $(AR) cru $(@) $(OBJS) #$(DPDK_LIBS) 56 | 57 | clean: 58 | rm -f $(OBJS) libix.a .depend 59 | 60 | dist-clean: clean 61 | rm *~ 62 | 63 | -------------------------------------------------------------------------------- /libix/buf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * buf.h = transmit data buffer management 57 | */ 58 | 59 | #pragma once 60 | 61 | #include 62 | #include 63 | 64 | #include 65 | #include 66 | 67 | 68 | #define BUF_SIZE 1460 * 4 69 | 70 | extern __thread struct mempool ixev_buf_pool; 71 | 72 | struct ixev_buf { 73 | uint32_t len; 74 | uint32_t pad; 75 | struct ixev_ref ref; 76 | char payload[BUF_SIZE]; 77 | }; 78 | 79 | static inline void ixev_buf_release(struct ixev_ref *ref) 80 | { 81 | struct ixev_buf *buf = container_of(ref, struct ixev_buf, ref); 82 | mempool_free(&ixev_buf_pool, buf); 83 | } 84 | 85 | /** 86 | * ixev_buf_alloc - allocates a buffer 87 | * 88 | * The initial refcount is set to one. 89 | * 90 | * Returns a buffer, or NULL if out of memory. 91 | */ 92 | static inline struct ixev_buf *ixev_buf_alloc(void) 93 | { 94 | struct ixev_buf *buf = mempool_alloc(&ixev_buf_pool); 95 | 96 | if (unlikely(!buf)) 97 | return NULL; 98 | 99 | buf->len = 0; 100 | buf->ref.cb = &ixev_buf_release; 101 | 102 | return buf; 103 | } 104 | 105 | /** 106 | * ixev_buf_store - store data inside a buffer 107 | * @buf: the buffer 108 | * @addr: the start address of the data 109 | * @len: the length of the data 110 | * 111 | * Returns the numbers of bytes successfully stored in the buffer, 112 | * or zero if the buffer is full. 113 | */ 114 | static inline size_t ixev_buf_store(struct ixev_buf *buf, void *addr, size_t len) 115 | { 116 | size_t avail = min(len, BUF_SIZE - buf->len); 117 | 118 | if (!avail) 119 | return 0; 120 | 121 | memcpy(&buf->payload[buf->len], addr, avail); 122 | buf->len += avail; 123 | 124 | return avail; 125 | } 126 | 127 | /** 128 | * ixev_is_buf_full - determines if the buffer is full 129 | * @buf: the buffer 130 | * 131 | * Returns true if the buffer is full, otherwise false. 132 | */ 133 | static inline bool ixev_is_buf_full(struct ixev_buf *buf) 134 | { 135 | return buf->len == BUF_SIZE; 136 | } 137 | 138 | 139 | -------------------------------------------------------------------------------- /libix/ixev_timer.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | #include "ixev_timer.h" 33 | #include "libix_syscall.h" 34 | 35 | int ixev_timer_init(struct ixev_timer *t, ixev_timer_handler_t h, void *arg) 36 | { 37 | t->handler = h; 38 | t->arg = arg; 39 | t->timer_id = sys_timer_init(t); 40 | 41 | return t->timer_id != -1; 42 | } 43 | 44 | int ixev_timer_add(struct ixev_timer *t, struct timeval tv) 45 | { 46 | uint64_t delay; 47 | 48 | delay = tv.tv_sec * 1000000 + tv.tv_usec; 49 | 50 | return sys_timer_ctl(t->timer_id, delay); 51 | } 52 | -------------------------------------------------------------------------------- /libix/ixev_timer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | #pragma once 33 | 34 | #include 35 | 36 | typedef void (*ixev_timer_handler_t)(void *arg); 37 | 38 | struct ixev_timer { 39 | ixev_timer_handler_t handler; 40 | void *arg; 41 | int timer_id; 42 | }; 43 | 44 | int ixev_timer_init(struct ixev_timer *t, ixev_timer_handler_t h, void *arg); 45 | 46 | int ixev_timer_add(struct ixev_timer *t, struct timeval tv); 47 | -------------------------------------------------------------------------------- /libix/libix_syscall.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * syscall.h - system call support 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | #include 32 | #include "syscall_raw.h" 33 | 34 | static inline int libix_sys_bpoll(struct bsys_desc *d, unsigned int nr) 35 | { 36 | printf("\n*******************IN LIBIX/SYS_BPOLL****************\n"); 37 | return (int) SYSCALL(SYS_BPOLL, d, nr); 38 | } 39 | 40 | static inline int sys_bcall(struct bsys_desc *d, unsigned int nr) 41 | { 42 | return (int) SYSCALL(SYS_BCALL, d, nr); 43 | } 44 | /* 45 | static inline void *sys_baddr(void) 46 | { 47 | return (struct bsys_arr *) SYSCALL(SYS_BADDR); 48 | } 49 | */ 50 | static inline int sys_mmap(void *addr, int nr, int size, int perm) 51 | { 52 | return (int) SYSCALL(SYS_MMAP, addr, nr, size, perm); 53 | } 54 | 55 | static inline int sys_unmap(void *addr, int nr, int size) 56 | { 57 | return (int) SYSCALL(SYS_MUNMAP, addr, nr, size); 58 | } 59 | 60 | static inline int sys_spawnmode(bool spawn_cores) 61 | { 62 | return (int) SYSCALL(SYS_SPAWNMODE, spawn_cores); 63 | } 64 | 65 | static inline int sys_nrcpus(void) 66 | { 67 | return (int) SYSCALL(SYS_NRCPUS); 68 | } 69 | 70 | static inline int sys_timer_init(void * addr) 71 | { 72 | return (int) SYSCALL(SYS_TIMER_INIT, addr); 73 | } 74 | 75 | static inline int sys_timer_ctl(int timer_id, uint64_t delay) 76 | { 77 | return (int) SYSCALL(SYS_TIMER_CTL, timer_id, delay); 78 | } 79 | -------------------------------------------------------------------------------- /libix/syscall.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * Copyright 2013-16 Board of Trustees of Stanford University 34 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 35 | * 36 | * Permission is hereby granted, free of charge, to any person obtaining a copy 37 | * of this software and associated documentation files (the "Software"), to deal 38 | * in the Software without restriction, including without limitation the rights 39 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | * copies of the Software, and to permit persons to whom the Software is 41 | * furnished to do so, subject to the following conditions: 42 | * 43 | * The above copyright notice and this permission notice shall be included in 44 | * all copies or substantial portions of the Software. 45 | * 46 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 52 | * THE SOFTWARE. 53 | */ 54 | 55 | /* 56 | * syscall.h - system call support 57 | */ 58 | 59 | #pragma once 60 | 61 | #include 62 | #include 63 | #include 64 | #include "syscall_raw.h" 65 | 66 | static inline int sys_bpoll(struct bsys_desc *d, unsigned int nr) 67 | { 68 | return (int) SYSCALL(SYS_BPOLL, d, nr); 69 | } 70 | 71 | static inline int sys_bcall(struct bsys_desc *d, unsigned int nr) 72 | { 73 | return (int) SYSCALL(SYS_BCALL, d, nr); 74 | } 75 | 76 | static inline void *sys_baddr(void) 77 | { 78 | return (struct bsys_arr *) SYSCALL(SYS_BADDR); 79 | } 80 | 81 | static inline int sys_mmap(void *addr, int nr, int size, int perm) 82 | { 83 | return (int) SYSCALL(SYS_MMAP, addr, nr, size, perm); 84 | } 85 | 86 | static inline int sys_unmap(void *addr, int nr, int size) 87 | { 88 | return (int) SYSCALL(SYS_MUNMAP, addr, nr, size); 89 | } 90 | 91 | static inline int sys_spawnmode(bool spawn_cores) 92 | { 93 | return (int) SYSCALL(SYS_SPAWNMODE, spawn_cores); 94 | } 95 | 96 | static inline int sys_nrcpus(void) 97 | { 98 | return (int) SYSCALL(SYS_NRCPUS); 99 | } 100 | 101 | static inline int sys_timer_init(void * addr) 102 | { 103 | return (int) SYSCALL(SYS_TIMER_INIT, addr); 104 | } 105 | 106 | static inline int sys_timer_ctl(int timer_id, uint64_t delay) 107 | { 108 | return (int) SYSCALL(SYS_TIMER_CTL, timer_id, delay); 109 | } 110 | -------------------------------------------------------------------------------- /reflex_nbd/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for the fixed_time kernel module 2 | # Adapted from the Dune Kernel Module 3 | # Adapted from the ipw2200 project. 4 | 5 | # We have limited control over the build process of merge files - the kernel 6 | # decides what compiler, how to do dependency tracking, etc. 7 | 8 | ifndef CONFIG_IX_REFLEX_MOD 9 | EXTERNAL_BUILD=y 10 | CONFIG_IX_REFLEX_MOD=m 11 | endif 12 | 13 | #WEIRDness. The name of the module can't be equal to any of the 14 | #constituent .c files, otherwise it only builds that one file. 15 | list-m := 16 | list-$(CONFIG_IX_REFLEX_MOD) += reflex 17 | obj-$(CONFIG_IX_REFLEX_MOD) += reflex.o 18 | reflex-objs := \ 19 | reflex_nbd.o \ 20 | 21 | # KSRC should be set to the path to your sources 22 | # modules are installed into KMISC 23 | KVER := $(shell uname -r) 24 | KSRC := /lib/modules/$(KVER)/build 25 | KMISC := /lib/modules/$(KVER)/kernel/drivers/misc 26 | KMISC_INC := /lib/modules/$(KVER)/include 27 | 28 | #SYSCALL_TBL = 0x$(shell $(EXTRACT) sys_call_table) 29 | #DO_FORK = 0x$(shell $(EXTRACT) do_fork) 30 | 31 | #Set to empty to print out whole commands for added targets 32 | V := @ 33 | 34 | ccflags-y := 35 | 36 | ccflags-y += -DDEBUG 37 | 38 | ccflags-y += -Wno-unused 39 | ccflags-y += -Wno-declaration-after-statement 40 | 41 | ccflags-y += -O3 42 | 43 | ccflags-y += -g 44 | 45 | #Static verification using smatch 46 | ifdef SMATCH 47 | CHECK := $(SMATCH) --project=kernel --full_path 48 | CC := $(SMATCHCC) 49 | CONFIG_DYNAMIC_DEBUG := n 50 | endif 51 | 52 | # KSRC_OUTPUT should be overridden if you are using a 2.6 kernel that 53 | # has it's output sent elsewhere via KBUILD_OUTPUT= or O= 54 | KSRC_OUTPUT := $(KSRC) 55 | 56 | PWD=$(shell pwd) 57 | 58 | VERFILE := $(KSRC_OUTPUT)/include/linux/version.h 59 | KERNELRELEASE := $(shell \ 60 | if [ -r $(VERFILE) ]; then \ 61 | (cat $(VERFILE); echo UTS_RELEASE) | \ 62 | $(CC) -I$(KSRC_OUTPUT) $(CFLAGS) -E - | \ 63 | tail -n 1 | \ 64 | xargs echo; \ 65 | else \ 66 | uname -r; \ 67 | fi) 68 | 69 | MODPATH := $(DESTDIR)/lib/modules/$(KERNELRELEASE) 70 | 71 | all: modules 72 | 73 | lint: clean 74 | $(V) $(CPPLINT) \ 75 | --extensions=c,h \ 76 | *.c *.h \ 77 | |& grep -v "Done processing" \ 78 | |& grep -v "Total errors found" \ 79 | || true 80 | 81 | clean: 82 | rm -f *.mod.c *.mod *.o *.ko .*.cmd .*.flags .lst *.lst *.order *.symvers 83 | rm -f .*.d *.d 84 | rm -rf $(PWD)/tmp 85 | # for file in *.{c,h}; do \ 86 | # sed -i -e "s:\ *$$::g" -e "s:\t*$$::g" $$file; \ 87 | done 88 | 89 | distclean: clean 90 | rm -f tags TAGS 91 | 92 | TMP=$(PWD)/tmp 93 | MODVERDIR=$(TMP)/.tmp_versions 94 | 95 | modules: 96 | mkdir -p $(MODVERDIR) 97 | # -cp $(IEEE80211_MODVERDIR)/*.mod $(MODVERDIR) 98 | ifdef ($(KSRC_OUTPUT)/.tmp_versions) 99 | -cp $(KSRC_OUTPUT)/.tmp_versions/*.mod $(MODVERDIR) 100 | endif 101 | ifeq ($(KSRC),$(KSRC_OUTPUT)) # We're not outputting elsewhere 102 | ifdef ($(KSRC)/.tmp_versions) 103 | -cp $(KSRC)/.tmp_versions/*.mod $(MODVERDIR) 104 | endif 105 | $(MAKE) -C $(KSRC) M=$(PWD) MODVERDIR=$(MODVERDIR) modules | tee $(TMP)/make.out 106 | @if grep -i "WARNING" $(TMP)/make.out > /dev/null; \ 107 | then echo "There were warnings. Aborting."; \ 108 | fi 109 | else # We've got a kernel with seperate output, copy the config, and use O= 110 | mkdir -p $(TMP) 111 | cp $(KSRC_OUTPUT)/.config $(TMP) 112 | $(MAKE) -C $(KSRC) M=$(PWD) MODVERDIR=$(MODVERDIR)) O=$(PWD)/tmp modules 113 | endif 114 | 115 | install: modules 116 | sudo install -d $(KMISC) 117 | sudo install -m 644 -c $(addsuffix .ko,$(list-m)) $(KMISC) 118 | sudo cp reflex.h /usr/local/include 119 | sudo /sbin/depmod -a ${KVER} 120 | ( sudo rmmod $(addsuffix .ko,$(list-m)) >& /dev/null ) || true 121 | sudo insmod $(addsuffix .ko,$(list-m)) 122 | sleep 1 123 | sudo chmod ugo+rw /dev/reflex 124 | 125 | uninstall: 126 | rm -rf $(addprefix $(KMISC),$(addsuffix .ko,$(list-m))) 127 | /sbin/depmod -a ${KVER} 128 | 129 | 130 | .PHONY: TAGS tags check_inc 131 | 132 | RCS_FIND_IGNORE := \( -name SCCS -o -name BitKeeper -o -name .svn -o -name CVS \) -prune -o 133 | define all-sources 134 | ( find . $(RCS_FIND_IGNORE) -name '*.[chS]' -print ) 135 | endef 136 | 137 | TAGS: 138 | $(all-sources) | etags - 139 | tags: 140 | rm -f $@ 141 | CTAGSF=`ctags --version | grep -i exuberant >/dev/null && echo "-I __initdata,__exitdata,EXPORT_SYMBOL,EXPORT_SYMBOL_NOVERS"`; \ 142 | $(all-sources) | xargs ctags $$CTAGSF -a 143 | 144 | -------------------------------------------------------------------------------- /reflex_nbd/flush_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | #include 33 | #include 34 | #include 35 | 36 | int main() 37 | { 38 | char my_write_str[] = "1234567890"; 39 | char my_read_str[100]; 40 | char my_filename[] = "/mnt/reflex/flush_test.txt"; 41 | int my_file_descriptor, close_err; 42 | 43 | /* Open the file. Clobber it if it exists. */ 44 | my_file_descriptor = open (my_filename, O_RDWR | O_CREAT | O_TRUNC); 45 | 46 | /* Write 10 bytes of data and make sure it's written */ 47 | write (my_file_descriptor, (void *) my_write_str, 10); 48 | printf("Do fsync()\n"); 49 | fsync (my_file_descriptor); 50 | printf("fsync done\n"); 51 | /* Seek the beginning of the file */ 52 | lseek (my_file_descriptor, 0, SEEK_SET); 53 | 54 | /* Read 10 bytes of data */ 55 | read (my_file_descriptor, (void *) my_read_str, 10); 56 | 57 | /* Terminate the data we've read with a null character */ 58 | my_read_str[10] = '\0'; 59 | 60 | printf ("String read = %s.\n", my_read_str); 61 | 62 | close (my_file_descriptor); 63 | 64 | return 0; 65 | } 66 | -------------------------------------------------------------------------------- /reflex_nbd/load_eth.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sudo ifconfig enp6s0f1 up 3 | sudo ifconfig enp6s0f1 10.79.6.131/24 4 | sudo ethtool -K enp6s0f1 lro off 5 | sudo ethtool -K enp6s0f1 gro off 6 | sudo ethtool -C enp6s0f1 rx-usecs 10 7 | -------------------------------------------------------------------------------- /reflex_nbd/load_reflex_mod.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #sudo rmmod flashix 3 | sudo insmod reflex.ko 4 | #sudo mke2fs -b 4096 -t ext4 /dev/flashix0 5 | sudo mount -t ext4 /dev/reflex0 /mnt/reflex 6 | sudo chmod a+rwx /mnt/reflex 7 | #touch /mnt/flashix/fio 8 | -------------------------------------------------------------------------------- /reflex_nbd/randread_remote.fio: -------------------------------------------------------------------------------- 1 | [global] 2 | rw=randread 3 | blocksize=${BLKSIZE} 4 | blockalign=${BLKSIZE} 5 | size=1G 6 | ioengine=libaio 7 | iodepth=${DEPTH} 8 | direct=1 9 | invalidate=1 10 | time_based 11 | runtime=20s 12 | numjobs=12 13 | group_reporting 14 | [job0] 15 | filename=/mnt/reflex/fio 16 | -------------------------------------------------------------------------------- /reflex_nbd/reflex_nbd.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015-2017, Stanford University 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * * Neither the name of the copyright holder nor the names of its 17 | * contributors may be used to endorse or promote products derived from 18 | * this software without specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | /* 33 | * 1999 Copyright (C) Pavel Machek, pavel@ucw.cz. This code is GPL. 34 | * 1999/11/04 Copyright (C) 1999 VMware, Inc. (Regis "HPReg" Duchesne) 35 | * Made nbd_end_request() use the io_request_lock 36 | * 2001 Copyright (C) Steven Whitehouse 37 | * New nbd_end_request() for compatibility with new linux block 38 | * layer code. 39 | * 2003/06/24 Louis D. Langholtz 40 | * Removed unneeded blksize_bits field from nbd_device struct. 41 | * Cleanup PARANOIA usage & code. 42 | * 2004/02/19 Paul Clements 43 | * Removed PARANOIA, plus various cleanup and comments 44 | */ 45 | #ifndef LINUX_REFLEX_H 46 | #define LINUX_REFLEX_H 47 | 48 | #include 49 | 50 | #include "../apps/reflex.h" 51 | 52 | enum { 53 | REFLEX_CMD_READ, 54 | REFLEX_CMD_WRITE, 55 | REFLEX_CMD_TRIM, 56 | REFLEX_CMD_FLUSH, 57 | }; 58 | 59 | struct request; 60 | 61 | struct reflex_cmd { 62 | struct request *rq; 63 | // struct reflex_queue *fq; 64 | }; 65 | 66 | struct reflex_queue { 67 | int index; 68 | long *reflex_reqs; 69 | struct mutex tx_lock; 70 | struct reflex_device *reflex_dev; 71 | struct socket *sock; 72 | struct task_struct *recvthread; 73 | }; 74 | 75 | struct reflex_device { 76 | int flags; 77 | int magic; 78 | 79 | struct gendisk *disk; 80 | int blksize; 81 | u64 bytesize; 82 | 83 | //blk-mq stuff 84 | struct request_queue *q; 85 | struct blk_mq_tag_set tag_set; 86 | 87 | struct reflex_queue *queues; 88 | unsigned int nr_queues; 89 | char disk_name[DISK_NAME_LEN]; 90 | 91 | }; 92 | 93 | /* 94 | * Memcached protocol support. 95 | * This part needs to be identical with the server definitions. 96 | */ 97 | 98 | #define CMD_GET 0x00 99 | #define CMD_SET 0x01 100 | #define CMD_SET_NO_ACK 0x02 101 | //#define CMD_SASL 0x21 102 | 103 | #define RESP_OK 0x00 104 | #define RESP_EINVAL 0x04 105 | //#define RESP_SASL_ERR 0x20 106 | 107 | #define REQ_PKT 0x80 108 | #define RESP_PKT 0x81 109 | #define MAX_EXTRA_LEN 8 110 | #define MAX_KEY_LEN 8 111 | 112 | #endif 113 | -------------------------------------------------------------------------------- /reflex_nbd/unload_reflex_mod.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sudo umount /mnt/reflex 3 | sudo rmmod reflex 4 | #sudo mke2fs -b 4096 -t ext4 /dev/flashix0 5 | 6 | 7 | -------------------------------------------------------------------------------- /run_reflex_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script to run ReFlex setup after machine reboot and start the ReFlex server 4 | 5 | sudo sh -c 'for i in /sys/devices/system/node/node*/hugepages/hugepages-2048kB/nr_hugepages; do echo 4096 > $i; done' 6 | sudo modprobe -r ixgbe 7 | sudo modprobe -r nvme 8 | sudo insmod deps/dune/kern/dune.ko 9 | sudo insmod deps/pcidma/pcidma.ko 10 | sudo ./dp/ix -- ./apps/reflex_server 11 | -------------------------------------------------------------------------------- /sample-aws-ec2.devmodel: -------------------------------------------------------------------------------- 1 | # sample.devmodel 2 | # Sample configuration file for Flash device 3 | # This file defines the request cost model for the Flash device used by ReFlex 4 | 5 | ############################################################################### 6 | # Request Costs 7 | ############################################################################### 8 | # ReFlex uses tokens as an I/O currency. We set a cost for a 4KB read request 9 | # and determine the cost of other request types (i.e., write requests) 10 | # and other request sizes relative to the cost of a 4KB read. 11 | # 12 | # Note: Default 4KB read cost of 100 tokens (instead of 1 token) is recommended 13 | # for more accurate tracking of fractions of tokens during scheduling. 14 | 15 | read_cost_4KB=100 # keep this default and adjust write cost in relation 16 | write_cost_4KB=700 # see Step 2 below for instructions on how to set 17 | 18 | ############################################################################### 19 | # Instructions for deriving request cost model: 20 | ############################################################################### 21 | # Profile the device by running local Flash tests for different rd/wr ratios 22 | # and request sizes and curve fit to find relative request costs. More details: 23 | # 24 | # You can use the SPDK example perf application to run local Flash tests. 25 | # Step 1: Run 4KB read-only test. Sweep IOPS and measure tail read latency. 26 | # Plot tail read latency vs. IOPS. 27 | # 28 | # Step 2: Run 4KB rd-wr tests with different rd/wr ratios. Plot tail read 29 | # latency vs. RdIOPS + WrIOPS. Find a weight_factor that when applied 30 | # to write IOs makes the latency-IOPS curves of different rd/wr ratios 31 | # overlap. In other words, plot tail read latency vs. weighted IOPS, 32 | # where weighted IOPS = RdIOPS + (weight_factor)(WrIOPS), and choose 33 | # weight_factor to make the curves for diff rd/wr ratios overlap. 34 | # This weight_factor represents the relative cost of a write IO. 35 | # Set write_cost4B = read_cost_4KB * weight_factor 36 | # 37 | # Step 3: Run tests for different request sizes. Plot tail read latency vs. 38 | # IOPS. Find a weight_factor that when applied to the non-4KB IOs 39 | # makes the latency-throughput curve overlap with the 4KB curve. 40 | # This weight_factor represents the relative cost for that IO size. 41 | # 42 | # In our experience, request cost tends to scale linearly with request size 43 | # for most devices. However, write vs. read cost is device specific. 44 | # Currently, we have only used ReFlex for 1KB and 4KB requests (which have 45 | # the same request cost on the SSD we used). 46 | 47 | 48 | ############################################################################### 49 | # Latency vs. Weighted IOPS curve (tokens/second limit for each latency SLO) 50 | ############################################################################### 51 | # The request cost model, derived using instructions above, is a graph of 52 | # tail read latency vs. weighted IOPS (measured in a currency of tokens/second). 53 | # 54 | # The request cost model tells you for each tail read latency service level 55 | # objective (SLO) the corresponding token rate limit. Enter the p95_latency_limit 56 | # and corresponding max_token_rate information in the config below for all latency 57 | # SLOs you care about. Below is an example. Add, delete, and/or modify the entries. 58 | # 59 | # If your device has significantly different behaviour for read-only workloads 60 | # compared to all other rd/wr mixed workloads, you can specify a token limit 61 | # to be used specifically when the total device load is read-only. 62 | # Use the parameter max_rdonly_token_rate for this. 63 | 64 | max_token_rate=40000000 # max token rate supported by device (no latency SLO) 65 | 66 | token_limits=( 67 | { 68 | p95_latency_limit : 500 #in us 69 | max_token_rate : 24000000 #in tokens 70 | max_rdonly_token_rate : 38000000 #in tokens 71 | }, 72 | { 73 | p95_latency_limit : 1000 #in us 74 | max_token_rate : 33000000 #in tokens 75 | max_rdonly_token_rate : 39000000 #in tokens 76 | }, 77 | { 78 | p95_latency_limit : 1500 #in us 79 | max_token_rate : 35000000 #in tokens 80 | max_rdonly_token_rate : 40000000 #in tokens 81 | }, 82 | { 83 | p95_latency_limit : 2000 #in us 84 | max_token_rate : 37000000 #in tokens 85 | max_rdonly_token_rate : 41000000 #in tokens 86 | }, 87 | { 88 | p95_latency_limit : 2500 #in us 89 | max_token_rate : 39000000 #in tokens 90 | max_rdonly_token_rate : 41000000 #in tokens 91 | }, 92 | { 93 | p95_latency_limit : 3000 #in us 94 | max_token_rate : 41000000 #in tokens 95 | max_rdonly_token_rate : 41500000 #in tokens 96 | } 97 | ) 98 | 99 | -------------------------------------------------------------------------------- /sample.devmodel: -------------------------------------------------------------------------------- 1 | # sample.devmodel 2 | # Sample configuration file for Flash device 3 | # This file defines the request cost model for the Flash device used by ReFlex 4 | 5 | ############################################################################### 6 | # Request Costs 7 | ############################################################################### 8 | # ReFlex uses tokens as an I/O currency. We set a cost for a 4KB read request 9 | # and determine the cost of other request types (i.e., write requests) 10 | # and other request sizes relative to the cost of a 4KB read. 11 | # 12 | # Note: Default 4KB read cost of 100 tokens (instead of 1 token) is recommended 13 | # for more accurate tracking of fractions of tokens during scheduling. 14 | 15 | read_cost_4KB=100 # keep this default and adjust write cost in relation 16 | write_cost_4KB=1000 # see Step 2 below for instructions on how to set 17 | 18 | ############################################################################### 19 | # Instructions for deriving request cost model: 20 | ############################################################################### 21 | # Profile the device by running local Flash tests for different rd/wr ratios 22 | # and request sizes and curve fit to find relative request costs. More details: 23 | # 24 | # You can use the SPDK example perf application to run local Flash tests. 25 | # Step 1: Run 4KB read-only test. Sweep IOPS and measure tail read latency. 26 | # Plot tail read latency vs. IOPS. 27 | # 28 | # Step 2: Run 4KB rd-wr tests with different rd/wr ratios. Plot tail read 29 | # latency vs. RdIOPS + WrIOPS. Find a weight_factor that when applied 30 | # to write IOs makes the latency-IOPS curves of different rd/wr ratios 31 | # overlap. In other words, plot tail read latency vs. weighted IOPS, 32 | # where weighted IOPS = RdIOPS + (weight_factor)(WrIOPS), and choose 33 | # weight_factor to make the curves for diff rd/wr ratios overlap. 34 | # This weight_factor represents the relative cost of a write IO. 35 | # Set write_cost4B = read_cost_4KB * weight_factor 36 | # 37 | # Step 3: Run tests for different request sizes. Plot tail read latency vs. 38 | # IOPS. Find a weight_factor that when applied to the non-4KB IOs 39 | # makes the latency-throughput curve overlap with the 4KB curve. 40 | # This weight_factor represents the relative cost for that IO size. 41 | # 42 | # In our experience, request cost tends to scale linearly with request size 43 | # for most devices. However, write vs. read cost is device specific. 44 | # Currently, we have only used ReFlex for 1KB and 4KB requests (which have 45 | # the same request cost on the SSD we used). 46 | 47 | 48 | ############################################################################### 49 | # Latency vs. Weighted IOPS curve (tokens/second limit for each latency SLO) 50 | ############################################################################### 51 | # The request cost model, derived using instructions above, is a graph of 52 | # tail read latency vs. weighted IOPS (measured in a currency of tokens/second). 53 | # 54 | # The request cost model tells you for each tail read latency service level 55 | # objective (SLO) the corresponding token rate limit. Enter the p95_latency_limit 56 | # and corresponding max_token_rate information in the config below for all latency 57 | # SLOs you care about. Below is an example. Add, delete, and/or modify the entries. 58 | # 59 | # If your device has significantly different behaviour for read-only workloads 60 | # compared to all other rd/wr mixed workloads, you can specify a token limit 61 | # to be used specifically when the total device load is read-only. 62 | # Use the parameter max_rdonly_token_rate for this. 63 | 64 | max_token_rate=100000000 # max token rate supported by device (no latency SLO) 65 | 66 | token_limits=( 67 | { 68 | p95_latency_limit : 500 #in us 69 | max_token_rate : 36000000 #in tokens 70 | max_rdonly_token_rate : 85000000 #in tokens 71 | }, 72 | { 73 | p95_latency_limit : 1000 #in us 74 | max_token_rate : 50000000 #in tokens 75 | max_rdonly_token_rate : 95000000 #in tokens 76 | }, 77 | { 78 | p95_latency_limit : 1500 #in us 79 | max_token_rate : 52000000 #in tokens 80 | max_rdonly_token_rate : 98000000 #in tokens 81 | }, 82 | { 83 | p95_latency_limit : 2000 #in us 84 | max_token_rate : 57000000 #in tokens 85 | max_rdonly_token_rate : 100000000 #in tokens 86 | }, 87 | { 88 | p95_latency_limit : 2500 #in us 89 | max_token_rate : 75000000 #in tokens 90 | max_rdonly_token_rate : 100000000 #in tokens 91 | }, 92 | { 93 | p95_latency_limit : 3000 #in us 94 | max_token_rate : 80000000 #in tokens 95 | max_rdonly_token_rate : 100000000 #in tokens 96 | } 97 | ) 98 | 99 | --------------------------------------------------------------------------------