├── .gitignore ├── .gitmodules ├── LICENSE ├── Makefile ├── README.md ├── apps ├── Makefile ├── echoclient.c └── echoserver.c ├── cp └── ixcp.py ├── dp ├── Makefile ├── core │ ├── cfg.c │ ├── control_plane.c │ ├── cpu.c │ ├── dir.mk │ ├── dpdk.c │ ├── ethdev.c │ ├── ethfg.c │ ├── ethqueue.c │ ├── init.c │ ├── kstats.c │ ├── log.c │ ├── mbuf.c │ ├── mem.c │ ├── mempool.c │ ├── page.c │ ├── pci.c │ ├── perf.c │ ├── stats.c │ ├── syscall.c │ ├── tailqueue.c │ ├── timer.c │ ├── utimer.c │ └── vm.c ├── drivers │ ├── common.c │ ├── dir.mk │ ├── i40e.c │ └── ixgbe.c ├── ix.ld ├── lwip │ ├── dir.mk │ ├── inet_chksum.c │ ├── ip4_addr.c │ ├── memp_min.c │ ├── misc.c │ └── pbuf.c ├── net │ ├── arp.c │ ├── dir.mk │ ├── dump.c │ ├── icmp.c │ ├── ip.c │ ├── net.c │ ├── net.h │ ├── tcp.c │ ├── tcp_api.c │ ├── tcp_in.c │ ├── tcp_out.c │ └── udp.c └── sandbox │ ├── dir.mk │ ├── exec.c │ ├── main.c │ ├── sandbox.h │ ├── trap.c │ └── umm.c ├── inc ├── asm │ ├── chksum.h │ ├── cpu.h │ └── uaccess.h ├── ix │ ├── atomic.h │ ├── bitmap.h │ ├── byteorder.h │ ├── cfg.h │ ├── compiler.h │ ├── config.h │ ├── control_plane.h │ ├── cpu.h │ ├── dpdk.h │ ├── drivers.h │ ├── errno.h │ ├── ethdev.h │ ├── ethfg.h │ ├── ethqueue.h │ ├── hash.h │ ├── kstats.h │ ├── kstatvectors.h │ ├── list.h │ ├── lock.h │ ├── log.h │ ├── mbuf.h │ ├── mem.h │ ├── mempool.h │ ├── page.h │ ├── pci.h │ ├── perf.h │ ├── profiler.h │ ├── stats.h │ ├── stddef.h │ ├── syscall.h │ ├── tailqueue.h │ ├── timer.h │ ├── types.h │ ├── uaccess.h │ ├── utimer.h │ └── vm.h ├── lwip │ ├── arch │ │ ├── cc.h │ │ └── perf.h │ ├── ipv4 │ │ └── lwip │ │ │ ├── icmp.h │ │ │ ├── ip4.h │ │ │ └── ip4_addr.h │ ├── ipv6 │ │ └── lwip │ │ │ ├── ip6.h │ │ │ ├── ip6_addr.h │ │ │ └── nd6.h │ ├── lwip │ │ ├── arch.h │ │ ├── debug.h │ │ ├── def.h │ │ ├── err.h │ │ ├── inet_chksum.h │ │ ├── ip.h │ │ ├── ip_addr.h │ │ ├── mem.h │ │ ├── memp.h │ │ ├── memp_std.h │ │ ├── netif.h │ │ ├── opt.h │ │ ├── pbuf.h │ │ ├── snmp.h │ │ ├── stats.h │ │ ├── sys.h │ │ ├── tcp.h │ │ ├── tcp_impl.h │ │ └── timers.h │ └── lwipopts.h └── net │ ├── arp.h │ ├── ethernet.h │ ├── icmp.h │ ├── ip.h │ └── udp.h ├── ix.conf.sample ├── libix ├── Makefile ├── buf.h ├── ix.h ├── ixev.c ├── ixev.h ├── ixev_timer.c ├── ixev_timer.h ├── main.c ├── mem.c ├── mempool.c ├── mempool.h ├── syscall.h └── syscall_raw.h └── tools ├── Makefile └── ix-stats-show.c /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.a 3 | *.d 4 | *.o 5 | .depend 6 | 7 | /ix.conf 8 | /dp/ix 9 | /apps/echoclient 10 | /apps/echoserver 11 | /tools/ix-stats-show 12 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "deps/dune"] 2 | path = deps/dune 3 | url = https://github.com/ix-project/dune.git 4 | [submodule "deps/pcidma"] 5 | path = deps/pcidma 6 | url = https://github.com/ix-project/pcidma.git 7 | [submodule "deps/dpdk"] 8 | path = deps/dpdk 9 | url = http://dpdk.org/git/dpdk 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | IX Open Source License. 3 | 4 | Copyright 2013-16 Board of Trustees of Stanford University 5 | Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in 15 | all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | SUBDIRS = dp libix apps tools 23 | CLEANDIRS = $(SUBDIRS:%=clean-%) 24 | 25 | all: $(SUBDIRS) 26 | 27 | apps: libix 28 | 29 | $(SUBDIRS): 30 | $(MAKE) -C $@ 31 | 32 | clean: $(CLEANDIRS) 33 | 34 | style: 35 | astyle -A8 -T8 -p -U -H --suffix=~ -r -Q --exclude=deps --exclude=inc/lwip --exclude=dp/lwip --exclude=dp/net/tcp.c --exclude=dp/net/tcp_in.c --exclude=dp/net/tcp_out.c --exclude=dp/drivers/ixgbe.c '*.c' '*.h' 36 | 37 | style-check: 38 | $(eval TEST=$(shell astyle --dry-run --formatted -A8 -T8 -p -U -H --suffix=~ -r -Q --exclude=deps --exclude=inc/lwip --exclude=dp/lwip --exclude=dp/net/tcp.c --exclude=dp/net/tcp_in.c --exclude=dp/net/tcp_out.c --exclude=dp/drivers/ixgbe.c '*.c' '*.h' | grep Formatted)) 39 | @if [ -z "$(TEST)" ] ; then\ 40 | echo "success";\ 41 | exit 0;\ 42 | else\ 43 | echo "failed : $(TEST)";\ 44 | exit 1;\ 45 | fi 46 | 47 | $(CLEANDIRS): 48 | $(MAKE) -C $(@:clean-%=%) clean 49 | 50 | .PHONY: all clean style $(SUBDIRS) $(CLEANDIRS) 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Overview 2 | 3 | IX a protected dataplane operating system project from Stanford and EPFL. It provides event-driven applications with: 4 | * low-latency (including at the tail) 5 | * high-throughput 6 | * efficient use of resources (to support workload consolidation and energy proportionality) 7 | 8 | IX is licensed under an MIT-style license. See LICENSE. 9 | 10 | ## Requirements 11 | 12 | IX requires Intel DPDK and a supported Intel NIC: 13 | - Intel 82599 14 | - Intel X520 15 | - Intel X540 16 | 17 | For more details, check the [requirements page](https://github.com/ix-project/ix/wiki/Requirements) 18 | 19 | ## Setup Instructions 20 | 21 | There is currently no binary distribution of IX. You will therefore have to compile it from source. Additionally, you will need to fetch and compile the source dependencies: 22 | 23 | 1. fetch the dependencies: 24 | ``` 25 | git submodule update --init 26 | ``` 27 | 28 | 2. build the dependecies: 29 | ``` 30 | sudo chmod +r /boot/System.map-`uname -r` 31 | make -sj64 -C deps/dune 32 | make -sj64 -C deps/pcidma 33 | make -sj64 -C deps/dpdk config T=x86_64-native-linuxapp-gcc 34 | make -sj64 -C deps/dpdk 35 | 36 | ``` 37 | 38 | 3. build IX: 39 | ``` 40 | sudo apt-get install libconfig-dev libnuma-dev 41 | make -sj64 42 | ``` 43 | The resulting executable files are `cp/ixcp.py` for the IX control plane and `dp/ix` for the IX dataplane kernel. 44 | 45 | 4. Set up the environment: 46 | ``` 47 | cp ix.conf.sample ix.conf 48 | # modify at least host_addr, gateway_addr, devices, and cpu 49 | sudo sh -c 'for i in /sys/devices/system/node/node*/hugepages/hugepages-2048kB/nr_hugepages; do echo 4096 > $i; done' 50 | sudo insmod deps/dune/kern/dune.ko 51 | sudo insmod deps/pcidma/pcidma.ko 52 | sudo modprobe uio 53 | sudo insmod deps/dpdk/build/kmod/igb_uio.ko 54 | sudo ifdown INTERFACE 55 | sudo deps/dpdk/tools/dpdk_nic_bind.py -b igb_uio PCI_ADDRESS 56 | ``` 57 | 58 | 5. run the IX TCP echo server and check that it works. Make sure that at least the "Device" field in ix.conf matches your network card PCI bus address (try `lspci | grep -i eth` and look for the virtual adapter): 59 | 60 | ``` 61 | sudo ./dp/ix -- ./apps/echoserver 4 62 | ``` 63 | 64 | Then, try from another Linux host: 65 | ``` 66 | echo 123 | nc -vv 67 | ``` 68 | You should see the following output: 69 | ``` 70 | Connection to port [tcp/*] succeeded! 71 | 123 72 | ``` 73 | -------------------------------------------------------------------------------- /apps/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | CC = gcc 23 | CFLAGS = -Wall -g -MD -O3 -I../libix -I../inc -pthread 24 | 25 | APPS = echoserver echoclient 26 | 27 | all: $(APPS) 28 | 29 | $(APPS): ../libix/libix.a 30 | 31 | $(APPS): %: %.o 32 | $(CC) $(CFLAGS) $^ -o $@ 33 | 34 | clean: 35 | rm -f *.o *.d $(APPS) 36 | 37 | -include *.d 38 | -------------------------------------------------------------------------------- /apps/echoclient.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | #include 32 | #include 33 | 34 | static unsigned long count; 35 | static unsigned int msg_size; 36 | 37 | enum { 38 | CLIENT_MODE_RECV, 39 | CLIENT_MODE_SEND, 40 | }; 41 | 42 | struct client_conn { 43 | struct ixev_ctx ctx; 44 | struct ip_tuple id; 45 | int mode; 46 | size_t bytes_recvd; 47 | size_t bytes_sent; 48 | char *data; 49 | }; 50 | 51 | static struct client_conn *c; 52 | 53 | static void client_die(void) 54 | { 55 | ixev_close(&c->ctx); 56 | fprintf(stderr, "remote connection was closed\n"); 57 | exit(-1); 58 | } 59 | 60 | static void print_stats(void) 61 | { 62 | ssize_t ret; 63 | char buf; 64 | static unsigned long prv_count; 65 | 66 | if (count - prv_count > 640000 / msg_size) { 67 | prv_count = count; 68 | ret = read(STDIN_FILENO, &buf, 1); 69 | if (ret == 0) { 70 | fprintf(stderr, "Error: EOF on STDIN.\n"); 71 | exit(1); 72 | } else if (ret == -1 && errno == EAGAIN) { 73 | } else if (ret == -1) { 74 | perror("read"); 75 | exit(1); 76 | } else { 77 | printf("%d %ld\n", msg_size, count); 78 | fflush(stdout); 79 | } 80 | } 81 | } 82 | 83 | static void main_handler(struct ixev_ctx *ctx, unsigned int reason) 84 | { 85 | ssize_t ret; 86 | 87 | while (1) { 88 | if (c->mode == CLIENT_MODE_SEND) { 89 | ret = ixev_send_zc(ctx, &c->data[c->bytes_sent], 90 | msg_size - c->bytes_sent); 91 | if (ret <= 0) { 92 | if (ret != -EAGAIN) 93 | client_die(); 94 | return; 95 | } 96 | 97 | c->bytes_sent += ret; 98 | if (c->bytes_sent < msg_size) 99 | return; 100 | 101 | c->bytes_recvd = 0; 102 | ixev_set_handler(ctx, IXEVIN, &main_handler); 103 | c->mode = CLIENT_MODE_RECV; 104 | } else { 105 | ret = ixev_recv(ctx, &c->data[c->bytes_recvd], 106 | msg_size - c->bytes_recvd); 107 | if (ret <= 0) { 108 | if (ret != -EAGAIN) 109 | client_die(); 110 | return; 111 | } 112 | 113 | c->bytes_recvd += ret; 114 | if (c->bytes_recvd < msg_size) 115 | return; 116 | 117 | count++; 118 | print_stats(); 119 | 120 | c->bytes_sent = 0; 121 | ixev_set_handler(ctx, IXEVOUT, &main_handler); 122 | c->mode = CLIENT_MODE_SEND; 123 | } 124 | } 125 | } 126 | 127 | static struct ixev_ctx *client_accept(struct ip_tuple *id) 128 | { 129 | return NULL; 130 | } 131 | 132 | static void client_release(struct ixev_ctx *ctx) { } 133 | 134 | static void client_dialed(struct ixev_ctx *ctx, long ret) 135 | { 136 | if (ret) 137 | fprintf(stderr, "failed to connect, ret = %ld\n", ret); 138 | 139 | c->mode = CLIENT_MODE_SEND; 140 | c->bytes_sent = 0; 141 | 142 | puts("ready"); 143 | fflush(stdout); 144 | 145 | ixev_set_handler(ctx, IXEVOUT, &main_handler); 146 | main_handler(&c->ctx, IXEVOUT); 147 | } 148 | 149 | struct ixev_conn_ops stream_conn_ops = { 150 | .accept = &client_accept, 151 | .release = &client_release, 152 | .dialed = &client_dialed, 153 | }; 154 | 155 | static int parse_ip_addr(const char *str, uint32_t *addr) 156 | { 157 | unsigned char a, b, c, d; 158 | 159 | if (sscanf(str, "%hhu.%hhu.%hhu.%hhu", &a, &b, &c, &d) != 4) 160 | return -EINVAL; 161 | 162 | *addr = MAKE_IP_ADDR(a, b, c, d); 163 | return 0; 164 | } 165 | 166 | int main(int argc, char *argv[]) 167 | { 168 | int ret; 169 | int flags; 170 | 171 | c = malloc(sizeof(struct client_conn)); 172 | if (!c) 173 | exit(-1); 174 | 175 | if (argc != 4) { 176 | fprintf(stderr, "Usage: IP PORT MSG_SIZE\n"); 177 | return -1; 178 | } 179 | 180 | if (parse_ip_addr(argv[1], &c->id.dst_ip)) { 181 | fprintf(stderr, "Bad IP address '%s'", argv[1]); 182 | exit(1); 183 | } 184 | 185 | c->id.dst_port = atoi(argv[2]); 186 | 187 | msg_size = atoi(argv[3]); 188 | c->data = malloc(msg_size); 189 | if (!c->data) 190 | exit(-1); 191 | 192 | ixev_init(&stream_conn_ops); 193 | 194 | ret = ixev_init_thread(); 195 | if (ret) { 196 | fprintf(stderr, "unable to init IXEV\n"); 197 | exit(ret); 198 | } 199 | 200 | flags = fcntl(STDIN_FILENO, F_GETFL, 0); 201 | fcntl(STDIN_FILENO, F_SETFL, flags | O_NONBLOCK); 202 | 203 | ixev_dial(&c->ctx, &c->id); 204 | 205 | while (1) 206 | ixev_wait(); 207 | 208 | return 0; 209 | } 210 | -------------------------------------------------------------------------------- /dp/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | # A Makefile for IX. 23 | # 24 | # path for this Makefile to work properly. 25 | 26 | DPDK = ../deps/dpdk 27 | DUNE = ../deps/dune 28 | PCIDMA = ../deps/pcidma 29 | DPDK_INC = -I$(DPDK)/build/include -I$(DPDK)/lib/librte_eal/common -I$(DPDK)/drivers/net/ixgbe -I$(DPDK)/drivers/net/i40e 30 | INC = -I../inc -I$(DUNE)/libdune -I../inc/lwip -I../inc/lwip/ipv4 -I../inc/lwip/ipv6 $(DPDK_INC) -include$(DUNE)/kern/dune.h 31 | INC += -I$(PCIDMA) 32 | CC = gcc 33 | CFLAGS = -g -Wall -fno-pie -fno-dwarf2-cfi-asm -fno-asynchronous-unwind-tables -O3 -mno-red-zone $(INC) -D__KERNEL__ $(EXTRA_CFLAGS) 34 | LD = gcc 35 | LDFLAGS = -T ix.ld -no-pie 36 | LDLIBS = -lrt -lpthread -lm -lnuma -ldl -lconfig 37 | 38 | ifneq ($(DEBUG),) 39 | CFLAGS += -DDEBUG 40 | endif 41 | 42 | ifneq ($(ENABLE_KSTATS),) 43 | CFLAGS += -DENABLE_KSTATS 44 | endif 45 | 46 | SRCS = 47 | DIRS = core drivers lwip net sandbox 48 | 49 | define register_dir 50 | SRCS += $(patsubst %, $(1)/%, $(2)) 51 | endef 52 | 53 | include $(patsubst %, %/dir.mk, $(DIRS)) 54 | 55 | %.generated.S: %.c 56 | $(CC) $(CFLAGS) -o $@ -S $< 57 | 58 | all: ix 59 | 60 | OBJS=$(subst .c,.o,$(SRCS)) 61 | 62 | DUNE_LIB=$(DUNE)/libdune/libdune.a 63 | 64 | depend: .depend 65 | 66 | .depend: $(SRCS) 67 | bash -c "cat $(foreach SRC,$(SRCS),<($(CC) $(CFLAGS) -MM -MT $(SRC:.c=.o) $(SRC))) > ./.depend" 68 | 69 | ifneq ($(MAKECMDGOALS),clean) 70 | -include .depend 71 | endif 72 | 73 | DPDK_LIBS= 74 | DPDK_LIBS+=-Wl,-whole-archive $(DPDK)/build/lib/librte_pmd_ixgbe.a -Wl,-no-whole-archive 75 | DPDK_LIBS+=-Wl,-whole-archive $(DPDK)/build/lib/librte_pmd_i40e.a -Wl,-no-whole-archive 76 | DPDK_LIBS+=$(DPDK)/build/lib/libethdev.a 77 | DPDK_LIBS+=$(DPDK)/build/lib/librte_mbuf.a 78 | DPDK_LIBS+=$(DPDK)/build/lib/librte_eal.a 79 | DPDK_LIBS+=$(DPDK)/build/lib/librte_mempool.a 80 | DPDK_LIBS+=$(DPDK)/build/lib/librte_ring.a 81 | 82 | ix: $(DEPENDENCY) $(OBJS) $(DUNE_LIB) ix.ld 83 | $(LD) $(LDFLAGS) -o ix $(OBJS) $(DUNE_LIB) $(DPDK_LIBS) $(LDLIBS) 84 | 85 | clean: 86 | rm -f $(OBJS) ix .depend 87 | 88 | dist-clean: clean 89 | rm *~ 90 | 91 | -------------------------------------------------------------------------------- /dp/core/control_plane.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * control_plane.c - control plane implementation 26 | */ 27 | 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | #include 36 | #include 37 | 38 | volatile struct cp_shmem *cp_shmem; 39 | 40 | DEFINE_PERCPU(volatile struct command_struct *, cp_cmd); 41 | 42 | double energy_unit; 43 | 44 | int cp_init(void) 45 | { 46 | int fd, ret; 47 | void *vaddr; 48 | 49 | fd = shm_open("/ix", O_RDWR | O_CREAT | O_TRUNC, 0660); 50 | if (fd == -1) 51 | return 1; 52 | 53 | ret = ftruncate(fd, sizeof(struct cp_shmem)); 54 | if (ret) 55 | return ret; 56 | 57 | vaddr = mmap(NULL, sizeof(struct cp_shmem), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 58 | if (vaddr == MAP_FAILED) 59 | return 1; 60 | 61 | cp_shmem = vaddr; 62 | 63 | bzero((void *)cp_shmem, sizeof(struct cp_shmem)); 64 | cp_shmem->cycles_per_us = cycles_per_us; 65 | 66 | return 0; 67 | } 68 | 69 | void cp_idle(void) 70 | { 71 | int fd, ret; 72 | char buf; 73 | 74 | percpu_get(cp_cmd)->cmd_id = CP_CMD_NOP; 75 | percpu_get(cp_cmd)->status = CP_STATUS_READY; 76 | percpu_get(cp_cmd)->cpu_state = CP_CPU_STATE_IDLE; 77 | fd = open((char *) percpu_get(cp_cmd)->idle.fifo, O_RDONLY); 78 | if (fd != -1) { 79 | ret = read(fd, &buf, 1); 80 | if (ret == -1) 81 | log_err("read on wakeup pipe returned -1 (errno=%d)\n", errno); 82 | close(fd); 83 | } 84 | percpu_get(cp_cmd)->cpu_state = CP_CPU_STATE_RUNNING; 85 | /* NOTE: reset timer position */ 86 | timer_init_cpu(); 87 | } 88 | -------------------------------------------------------------------------------- /dp/core/dir.mk: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | # Makefile for the core system 23 | 24 | SRC = ethdev.c ethfg.c ethqueue.c cfg.c control_plane.c cpu.c init.c log.c mbuf.c mem.c mempool.c page.c pci.c utimer.c syscall.c timer.c vm.c dpdk.c perf.c stats.c 25 | 26 | ifneq ($(ENABLE_KSTATS),) 27 | SRC += kstats.c tailqueue.c 28 | endif 29 | 30 | $(eval $(call register_dir, core, $(SRC))) 31 | 32 | -------------------------------------------------------------------------------- /dp/core/dpdk.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* For memmove and size_t */ 25 | #include 26 | 27 | /* For optind */ 28 | #include 29 | 30 | /* For struct sockaddr */ 31 | #include 32 | 33 | /* General DPDK includes */ 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | /* Defines from IX override DPDK */ 40 | #undef likely 41 | #undef unlikely 42 | 43 | /* IX includes */ 44 | #include 45 | #include 46 | #include 47 | 48 | struct rte_mempool *dpdk_pool; 49 | 50 | enum { 51 | DEV_DETACHED = 0, 52 | DEV_ATTACHED 53 | }; 54 | 55 | int dpdk_init(void) 56 | { 57 | int ret; 58 | /* -m stands for memory in MBs that DPDK will allocate. Must be enough 59 | * to accommodate the pool_size defined below. */ 60 | char cpu_id_str[8]; 61 | char *argv[] = { "./ix", "-m", "148", "-l", cpu_id_str }; 62 | const int pool_buffer_size = 0; 63 | const int pool_cache_size = 0; 64 | /* pool_size sets an implicit limit on cores * NICs that DPDK allows */ 65 | const int pool_size = 32768; 66 | 67 | /* We want to place DPDK in the desired NUMA node. Otherwise, its memory 68 | * allocations will fail. */ 69 | sprintf(cpu_id_str, "%d", CFG.cpu[0]); 70 | 71 | optind = 0; 72 | ret = rte_eal_init(sizeof(argv) / sizeof(argv[0]), argv); 73 | if (ret < 0) 74 | return ret; 75 | 76 | dpdk_pool = rte_pktmbuf_pool_create("mempool", pool_size, pool_cache_size, 0, pool_buffer_size, rte_socket_id()); 77 | if (dpdk_pool == NULL) 78 | panic("Cannot create DPDK pool\n"); 79 | 80 | return 0; 81 | } 82 | 83 | uint8_t rte_eth_dev_find_free_port(void) 84 | { 85 | unsigned i; 86 | 87 | for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 88 | if (rte_eth_devices[i].attached == DEV_DETACHED) 89 | return i; 90 | } 91 | return RTE_MAX_ETHPORTS; 92 | } 93 | -------------------------------------------------------------------------------- /dp/core/log.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * log.c - the logging system 26 | * 27 | * FIXME: Should we direct logs to a file? 28 | */ 29 | 30 | #include 31 | #include 32 | #include 33 | 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | #define MAX_LOG_LEN 1024 40 | 41 | __thread bool log_is_early_boot = true; 42 | 43 | int max_loglevel = LOG_DEBUG; 44 | 45 | void logk(int level, const char *fmt, ...) 46 | { 47 | va_list ptr; 48 | char buf[MAX_LOG_LEN]; 49 | time_t ts; 50 | off_t off = 0; 51 | 52 | if (level == LOG_CONT) { 53 | va_start(ptr, fmt); 54 | vprintf(fmt, ptr); 55 | va_end(ptr); 56 | 57 | return; 58 | } 59 | 60 | if (level > max_loglevel) 61 | return; 62 | 63 | if (!log_is_early_boot) { 64 | snprintf(buf, 9, "CPU %02d| ", percpu_get(cpu_id)); 65 | off = strlen(buf); 66 | } 67 | 68 | time(&ts); 69 | off += strftime(buf + off, 32, "%H:%M:%S ", localtime(&ts)); 70 | 71 | snprintf(buf + off, 6, "<%d>: ", level); 72 | off = strlen(buf); 73 | 74 | va_start(ptr, fmt); 75 | vsnprintf(buf + off, MAX_LOG_LEN - off, fmt, ptr); 76 | va_end(ptr); 77 | 78 | printf("%s", buf); 79 | } 80 | 81 | -------------------------------------------------------------------------------- /dp/core/mbuf.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * mbuf.c - buffer management for network packets 26 | * 27 | * TODO: add support for mapping into user-level address space... 28 | */ 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | 36 | /* Capacity should be at least RX queues per CPU * ETH_DEV_RX_QUEUE_SZ */ 37 | #define MBUF_CAPACITY (768*1024) 38 | 39 | static struct mempool_datastore mbuf_datastore; 40 | 41 | DEFINE_PERCPU(struct mempool, mbuf_mempool __attribute__((aligned(64)))); 42 | 43 | void mbuf_default_done(struct mbuf *m) 44 | { 45 | mbuf_free(m); 46 | } 47 | 48 | /** 49 | * mbuf_init_cpu - allocates the core-local mbuf region 50 | * 51 | * Returns 0 if successful, otherwise failure. 52 | */ 53 | 54 | int mbuf_init_cpu(void) 55 | { 56 | struct mempool *m = &percpu_get(mbuf_mempool); 57 | return mempool_create(m, &mbuf_datastore, MEMPOOL_SANITY_PERCPU, percpu_get(cpu_id)); 58 | } 59 | 60 | /** 61 | * mbuf_init - allocate global mbuf 62 | */ 63 | 64 | int mbuf_init(void) 65 | { 66 | int ret; 67 | struct mempool_datastore *m = &mbuf_datastore; 68 | BUILD_ASSERT(sizeof(struct mbuf) <= MBUF_HEADER_LEN); 69 | 70 | ret = mempool_create_datastore(m, MBUF_CAPACITY, MBUF_LEN, 1, MEMPOOL_DEFAULT_CHUNKSIZE, "mbuf"); 71 | if (ret) { 72 | assert(0); 73 | return ret; 74 | } 75 | ret = mempool_pagemem_map_to_user(m); 76 | if (ret) { 77 | assert(0); 78 | mempool_pagemem_destroy(m); 79 | return ret; 80 | } 81 | return 0; 82 | } 83 | 84 | /** 85 | * mbuf_exit_cpu - frees the core-local mbuf region 86 | */ 87 | void mbuf_exit_cpu(void) 88 | { 89 | mempool_pagemem_destroy(&mbuf_datastore); 90 | } 91 | 92 | -------------------------------------------------------------------------------- /dp/core/page.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * page.c - page-level memory management 26 | * 27 | * FIXME: Performance and address-space conservation could be 28 | * improved by maintaining free-lists of previously allocated 29 | * pages. 30 | * 31 | * FIXME: Need to add support for 1GB pages. May also consider 32 | * 4KB pages. 33 | */ 34 | 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | 41 | #define NUM_PAGES PGN_2MB(MEM_USER_START - MEM_PHYS_BASE_ADDR) 42 | 43 | struct page_ent page_tbl[NUM_PAGES]; 44 | DEFINE_PERCPU(int32_t, page_refs[NUM_PAGES]); 45 | 46 | static atomic64_t page_pos = ATOMIC_INIT(MEM_PHYS_BASE_ADDR); 47 | 48 | static inline struct page_ent *addr_to_page_ent(void *addr) 49 | { 50 | return &page_tbl[PAGE_NUM(addr)]; 51 | } 52 | 53 | /** 54 | * __page_put_slow - the slow path for decrementing page refernces 55 | * @addr: the address 56 | * 57 | * This function actually frees the page (if possible). 58 | */ 59 | void __page_put_slow(void *addr) 60 | { 61 | bool no_refs; 62 | struct page_ent *ent = addr_to_page_ent(addr); 63 | 64 | no_refs = atomic_dec_and_test(&ent->refcnt); 65 | 66 | /* can we free the page yet? */ 67 | if (!no_refs || !(ent->flags & PAGE_FLAG_CAN_FREE)) 68 | return; 69 | 70 | mem_free_page((void *) PGADDR_2MB(addr), PGSIZE_2MB); 71 | } 72 | 73 | /** 74 | * page_alloc_contig_on_node - allocates a guest-physically contiguous set of 2MB pages 75 | * @nr: the number of pages 76 | * @numa_node: the numa node 77 | * 78 | * Returns an address, or NULL if fail. 79 | */ 80 | void *page_alloc_contig_on_node(unsigned int nr, int numa_node) 81 | { 82 | int ret, i; 83 | 84 | void *base = (void *) atomic64_fetch_and_add(&page_pos, nr * PGSIZE_2MB); 85 | 86 | if ((uintptr_t) base >= MEM_USER_START) 87 | return NULL; 88 | 89 | 90 | base = __mem_alloc_pages_onnode(base, nr, PGSIZE_2MB, numa_node); 91 | if (base == MAP_FAILED) 92 | return NULL; 93 | 94 | for (i = 0; i < nr; i++) { 95 | 96 | void *pos = (void *)((uintptr_t) base + i * PGSIZE_2MB); 97 | struct page_ent *ent = addr_to_page_ent(pos); 98 | *((int *) pos) = 0; /* force a fault */ 99 | ret = mem_lookup_page_machine_addr(pos, PGSIZE_2MB, &ent->maddr); 100 | if (ret) { 101 | mem_free_pages(base, nr, PGSIZE_2MB); 102 | log_err("page: failed to get machine address for %p\n", pos); 103 | return NULL; 104 | } 105 | } 106 | 107 | return base; 108 | } 109 | 110 | /** 111 | * page_free - frees a page 112 | * @addr: the address of (or within) the page 113 | */ 114 | void page_free(void *addr) 115 | { 116 | struct page_ent *ent = addr_to_page_ent(addr); 117 | 118 | ent->flags |= PAGE_FLAG_WILL_FREE; 119 | 120 | /* FIXME: need RCU infrastructure to complete this. */ 121 | } 122 | 123 | /** 124 | * page_free_contig - frees a contiguous group of pages 125 | * @addr: the address of (or within) the first page 126 | * @nr: the number of pages 127 | */ 128 | void page_free_contig(void *addr, unsigned int nr) 129 | { 130 | int i; 131 | 132 | for (i = 0; i < nr; i++) { 133 | page_free((void *)((uintptr_t) addr + PGSIZE_2MB * i)); 134 | } 135 | } 136 | 137 | -------------------------------------------------------------------------------- /dp/core/perf.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | 8 | static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags) 9 | { 10 | return syscall(__NR_perf_event_open, hw_event, pid, cpu, group_fd, flags); 11 | } 12 | 13 | long read_perf_event(int fd) 14 | { 15 | int ret; 16 | long long value; 17 | 18 | ret = read(fd, &value, sizeof(long)); 19 | if (ret != sizeof(long)) 20 | value = -1; 21 | ioctl(fd, PERF_EVENT_IOC_RESET, 0); 22 | return value; 23 | } 24 | 25 | int init_perf_event(struct perf_event_attr *attr) 26 | { 27 | int fd; 28 | 29 | attr->size = sizeof(struct perf_event_attr); 30 | fd = perf_event_open(attr, 0, -1, -1, 0); 31 | ioctl(fd, PERF_EVENT_IOC_RESET, 0); 32 | ioctl(fd, PERF_EVENT_IOC_ENABLE, 0); 33 | return fd; 34 | } 35 | -------------------------------------------------------------------------------- /dp/core/stats.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | #if CONFIG_STATS 15 | 16 | static DEFINE_PERCPU(int, llc_load_misses_fd); 17 | 18 | struct ix_stats *stats; 19 | 20 | int stats_init(void) 21 | { 22 | int fd, ret; 23 | void *vaddr; 24 | 25 | fd = shm_open("/ix-stats", O_RDWR | O_CREAT | O_TRUNC, 0660); 26 | if (fd == -1) 27 | return 1; 28 | 29 | ret = ftruncate(fd, sizeof(struct ix_stats)); 30 | if (ret) 31 | return ret; 32 | 33 | vaddr = mmap(NULL, sizeof(struct ix_stats), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 34 | if (vaddr == MAP_FAILED) 35 | return 1; 36 | 37 | stats = vaddr; 38 | 39 | bzero((void *)stats, sizeof(struct ix_stats)); 40 | 41 | stats->cpus = CFG.num_cpus; 42 | return 0; 43 | } 44 | 45 | int stats_init_cpu(void) 46 | { 47 | struct perf_event_attr llc_load_misses_attr = { 48 | .type = PERF_TYPE_HW_CACHE, 49 | .config = (PERF_COUNT_HW_CACHE_LL) | 50 | (PERF_COUNT_HW_CACHE_OP_READ << 8) | 51 | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16), 52 | }; 53 | 54 | percpu_get(llc_load_misses_fd) = init_perf_event(&llc_load_misses_attr); 55 | 56 | return 0; 57 | } 58 | 59 | void stats_check_reset(void) 60 | { 61 | struct ix_stats_percpu *s = &stats->percpu[percpu_get(cpu_nr)]; 62 | 63 | if (s->poll) { 64 | s->llc_load_misses = read_perf_event(percpu_get(llc_load_misses_fd)); 65 | asm volatile("":::"memory"); 66 | s->poll = 0; 67 | } 68 | 69 | if (s->reset) 70 | bzero(stats->percpu, sizeof(stats->percpu)); 71 | } 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /dp/core/tailqueue.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | 25 | /* 26 | * tailqueue.c -- measurement of the tail queue of a distribution 27 | * used to track the latency in delivery of packets 28 | * 29 | * 30 | */ 31 | 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | #include 38 | 39 | #define OVERLAP_FACTOR 2 40 | #define NUM_BUCKETS (10*OVERLAP_FACTOR) 41 | #define NUM_LEVELS 5 42 | #define GRANULARITY_0 (10) /* 10 microseconds */ 43 | 44 | 45 | typedef struct tailqueue { 46 | uint32_t count; 47 | uint64_t min; 48 | uint64_t max; 49 | uint32_t gran[NUM_LEVELS][NUM_BUCKETS]; 50 | uint32_t overflow; 51 | } tailqueue; 52 | 53 | typedef struct taildistr taildistr; 54 | 55 | 56 | 57 | void tailqueue_addsample(struct tailqueue *tq, 58 | uint64_t t_us) 59 | { 60 | int i; 61 | uint64_t x; 62 | uint64_t gran = GRANULARITY_0 * pow(10, NUM_LEVELS - 1); 63 | 64 | 65 | x = t_us / gran; 66 | if (x >= NUM_BUCKETS) { 67 | 68 | tq->overflow++; 69 | } else { 70 | for (i = NUM_LEVELS - 1; i >= 0; i--) { 71 | tq->gran[i][x]++; 72 | gran = gran / 10; 73 | x = t_us / gran; 74 | if (x >= NUM_BUCKETS) 75 | break; 76 | } 77 | } 78 | if (tq->count == 0) { 79 | tq->min = t_us; 80 | tq->max = t_us; 81 | } else { 82 | if (tq->min > t_us) 83 | tq->min = t_us; 84 | if (tq->max < t_us) 85 | tq->max = t_us; 86 | } 87 | tq->count++; 88 | } 89 | 90 | 91 | 92 | void tailqueue_calcnines(struct tailqueue *tq, 93 | struct taildistr *td, 94 | int reset) 95 | { 96 | 97 | uint64_t above; 98 | uint64_t threshold; 99 | int nines = MAX_NINES; 100 | int cur_level, cur_bucket; 101 | bzero(td, sizeof(*td)); 102 | if (tq->count == 0) { 103 | return; 104 | } 105 | td->count = tq->count; 106 | td->min = tq->min; 107 | td->max = tq->max; 108 | 109 | /* 110 | * start with the max, starting with overflow; go down until you hit the 99.99, 99.9, 99% marks 111 | */ 112 | 113 | if (tq->overflow) { 114 | above = tq->overflow; 115 | for (; nines >= MIN_NINES; nines--) { 116 | threshold = tq->count / pow(10, nines); 117 | if (above >= threshold) { 118 | td->nines[nines] = tq->max; 119 | } else { 120 | break; 121 | } 122 | } 123 | cur_level = NUM_LEVELS - 1; 124 | cur_bucket = NUM_BUCKETS - 1; 125 | } else { 126 | uint64_t gran = GRANULARITY_0 * pow(10, NUM_LEVELS - 1); 127 | cur_bucket = tq->max / gran; 128 | cur_level = NUM_LEVELS - 1; 129 | above = 0; 130 | } 131 | 132 | for (; nines >= MIN_NINES; nines--) { 133 | threshold = tq->count / pow(10, nines); 134 | while (cur_level && cur_bucket) { 135 | if (cur_level && cur_bucket < OVERLAP_FACTOR) { 136 | cur_level--; 137 | cur_bucket = NUM_BUCKETS - 1; 138 | } 139 | above += tq->gran[cur_level][cur_bucket]; 140 | if (above >= threshold) { 141 | td->nines[nines] = GRANULARITY_0 * (pow(10, cur_level) + cur_bucket) ; 142 | //printf("nines=%d above = %lu threshold=%lu level=%d bucket=%d val=%lu\n", 143 | // nines,above,threshold,cur_level,cur_bucket,td->nines[nines]); 144 | break; 145 | } 146 | cur_bucket--; 147 | } 148 | } 149 | 150 | 151 | if (reset) { 152 | bzero(tq, sizeof(*tq)); 153 | } 154 | } 155 | 156 | 157 | 158 | /* 159 | * debugging 160 | */ 161 | 162 | #ifdef notdef 163 | main(int argc, char **argv) 164 | { 165 | tailqueue tq; 166 | taildistr td; 167 | int i; 168 | 169 | bzero(&tq, sizeof(tq)); 170 | bzero(&td, sizeof(td)); 171 | 172 | int n = atoi(argv[1]); 173 | printf("exp. distribution with %d samples \n", n); 174 | 175 | for (i = 0; i < n; i++) { 176 | float u = (random() % 1000000) / 1000000.0; 177 | float val = -log(u); 178 | uint64_t x = (uint64_t)(val * 1000000.0 * .5); 179 | printf("rand %5.3f %5.3f %lu \n", u, val, x); 180 | tailqueue_addsample(&tq, x); 181 | } 182 | 183 | for (i = 0; i < NUM_BUCKETS; i++) { 184 | printf("bucket %3d : ", i); 185 | int l; 186 | for (l = 0; l < NUM_LEVELS; l++) { 187 | printf(" %4d | ", tq.gran[l][i]); 188 | } 189 | printf("\n"); 190 | } 191 | 192 | printf("overflow: %d\n", tq.overflow); 193 | tailqueue_calcnines(&tq, &td, 0); 194 | printf("count = %u min = %lu max = %lu \n", n, td.min, td.max); 195 | for (i = MIN_NINES; i <= MAX_NINES; i++) { 196 | printf(" %dnines %lu\n", i, td.nines[i]); 197 | } 198 | } 199 | 200 | #endif 201 | 202 | -------------------------------------------------------------------------------- /dp/core/utimer.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | /* max number of supported user level timers */ 5 | #define UTIMER_COUNT 32 6 | 7 | struct utimer { 8 | struct timer t; 9 | void *cookie; 10 | }; 11 | 12 | struct utimer_list { 13 | struct utimer arr[UTIMER_COUNT]; 14 | }; 15 | 16 | DEFINE_PERCPU(struct utimer_list, utimers); 17 | 18 | void generic_handler(struct timer *t, struct eth_fg *unused) 19 | { 20 | struct utimer *ut; 21 | ut = container_of(t, struct utimer, t); 22 | usys_timer((unsigned long) ut->cookie); 23 | } 24 | 25 | static int find_available(struct utimer_list *tl) 26 | { 27 | static int next; 28 | 29 | if (next >= UTIMER_COUNT) 30 | return -1; 31 | 32 | return next++; 33 | } 34 | 35 | int utimer_init(struct utimer_list *tl, void *udata) 36 | { 37 | struct utimer *ut; 38 | int index; 39 | 40 | index = find_available(tl); 41 | if (index < 0) 42 | return -1; 43 | 44 | ut = &tl->arr[index]; 45 | ut->cookie = udata; 46 | timer_init_entry(&ut->t, generic_handler); 47 | 48 | return index; 49 | } 50 | 51 | int utimer_arm(struct utimer_list *tl, int timer_id, uint64_t delay) 52 | { 53 | struct timer *t; 54 | t = &tl->arr[timer_id].t; 55 | return timer_add(t, NULL, delay); 56 | } 57 | -------------------------------------------------------------------------------- /dp/drivers/dir.mk: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | SRC = ixgbe.c i40e.c common.c 23 | $(eval $(call register_dir, drivers, $(SRC))) 24 | 25 | -------------------------------------------------------------------------------- /dp/lwip/dir.mk: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | # Temporary makefile for LWIP support code 23 | 24 | SRC = inet_chksum.c ip4_addr.c memp_min.c misc.c pbuf.c 25 | $(eval $(call register_dir, lwip, $(SRC))) 26 | 27 | -------------------------------------------------------------------------------- /dp/lwip/memp_min.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | #include "lwip/memp.h" 25 | #include "lwip/tcp_impl.h" 26 | #include "lwip/timers.h" 27 | 28 | #define MEMP_ALIGN_SIZE(x) (LWIP_MEM_ALIGN_SIZE(x)) 29 | 30 | const u16_t memp_sizes[MEMP_MAX] = { 31 | #define LWIP_MEMPOOL(name,num,size,desc) LWIP_MEM_ALIGN_SIZE(size), 32 | #include 33 | }; 34 | -------------------------------------------------------------------------------- /dp/lwip/misc.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | 25 | 26 | #include 27 | #include 28 | #include 29 | 30 | #include 31 | 32 | #include 33 | #include 34 | #include 35 | 36 | static struct netif { 37 | char unused[32]; 38 | } netif; 39 | 40 | struct ip_globals 41 | { 42 | char unused[20]; 43 | struct ip_addr current_iphdr_src; 44 | struct ip_addr current_iphdr_dest; 45 | }; 46 | 47 | void tcp_input(struct eth_fg *cur_fg,struct pbuf *p, struct ip_addr *src, struct ip_addr *dest); 48 | 49 | //DEFINE_PERCPU(struct ip_globals, ip_data); 50 | 51 | 52 | struct netif *ip_route(struct ip_addr *dest) 53 | { 54 | return &netif; 55 | } 56 | 57 | void tcp_input_tmp(struct eth_fg *cur_fg, struct mbuf *pkt, struct ip_hdr *iphdr, void *tcphdr) 58 | { 59 | struct pbuf *pbuf; 60 | 61 | pbuf = pbuf_alloc(PBUF_RAW, ntoh16(iphdr->len) - iphdr->header_len * 4, PBUF_ROM); 62 | pbuf->payload = tcphdr; 63 | pbuf->mbuf = pkt; 64 | // percpu_get(ip_data).current_iphdr_dest.addr = iphdr->dst_addr.addr; 65 | // percpu_get(ip_data).current_iphdr_src.addr = iphdr->src_addr.addr; 66 | tcp_input(cur_fg,pbuf, &iphdr->src_addr,&iphdr->dst_addr); 67 | } 68 | 69 | 70 | static struct mempool_datastore pbuf_ds; 71 | static struct mempool_datastore pbuf_with_payload_ds; 72 | static struct mempool_datastore tcp_pcb_ds; 73 | static struct mempool_datastore tcp_seg_ds; 74 | 75 | DEFINE_PERCPU(struct mempool, pbuf_mempool __attribute__ ((aligned (64)))); 76 | DEFINE_PERCPU(struct mempool, pbuf_with_payload_mempool __attribute__ ((aligned (64)))); 77 | DEFINE_PERCPU(struct mempool, tcp_pcb_mempool __attribute__ ((aligned (64)))); 78 | DEFINE_PERCPU(struct mempool, tcp_pcb_listen_mempool __attribute__ ((aligned (64)))); 79 | DEFINE_PERCPU(struct mempool, tcp_seg_mempool __attribute__ ((aligned (64)))); 80 | 81 | #define MEMP_SIZE (256*1024) 82 | #define PBUF_CAPACITY (768*1024) 83 | 84 | #define PBUF_WITH_PAYLOAD_SIZE 4096 85 | 86 | static int init_mempool(struct mempool_datastore *m, int nr_elems, size_t elem_len, const char *prettyname) 87 | { 88 | return mempool_create_datastore(m, nr_elems, elem_len, 0, MEMPOOL_DEFAULT_CHUNKSIZE,prettyname); 89 | } 90 | 91 | int memp_init(void) 92 | { 93 | if (init_mempool(&pbuf_ds, PBUF_CAPACITY, memp_sizes[MEMP_PBUF],"pbuf")) 94 | return 1; 95 | 96 | if (init_mempool(&pbuf_with_payload_ds, MEMP_SIZE, PBUF_WITH_PAYLOAD_SIZE,"pbuf_payload")) 97 | return 1; 98 | 99 | if (init_mempool(&tcp_pcb_ds, MEMP_SIZE, memp_sizes[MEMP_TCP_PCB],"tcp_pcb")) 100 | return 1; 101 | 102 | if (init_mempool(&tcp_seg_ds, MEMP_SIZE, memp_sizes[MEMP_TCP_SEG],"tcp_seg")) 103 | return 1; 104 | return 0; 105 | } 106 | 107 | int memp_init_cpu(void) 108 | { 109 | int cpu = percpu_get(cpu_id); 110 | 111 | if (mempool_create(&percpu_get(pbuf_mempool),&pbuf_ds,MEMPOOL_SANITY_PERCPU, cpu)) 112 | return 1; 113 | 114 | if (mempool_create(&percpu_get(pbuf_with_payload_mempool), &pbuf_with_payload_ds, MEMPOOL_SANITY_PERCPU, cpu)) 115 | return 1; 116 | 117 | if (mempool_create(&percpu_get(tcp_pcb_mempool), &tcp_pcb_ds, MEMPOOL_SANITY_PERCPU, cpu)) 118 | return 1; 119 | 120 | if (mempool_create(&percpu_get(tcp_seg_mempool), &tcp_seg_ds, MEMPOOL_SANITY_PERCPU, cpu)) 121 | return 1; 122 | 123 | return 0; 124 | } 125 | 126 | void *mem_malloc(size_t size) 127 | { 128 | LWIP_ASSERT("mem_malloc", size <= PBUF_WITH_PAYLOAD_SIZE); 129 | return mempool_alloc(&percpu_get(pbuf_with_payload_mempool)); 130 | } 131 | 132 | void mem_free(void *ptr) 133 | { 134 | mempool_free(&percpu_get(pbuf_with_payload_mempool), ptr); 135 | } 136 | -------------------------------------------------------------------------------- /dp/net/dir.mk: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | # Makefile for network module 23 | 24 | SRC = arp.c dump.c icmp.c ip.c net.c tcp.c tcp_in.c tcp_out.c \ 25 | tcp_api.c udp.c 26 | $(eval $(call register_dir, net, $(SRC))) 27 | 28 | -------------------------------------------------------------------------------- /dp/net/dump.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * dump.c - logs network headers in a human-readable format 26 | * 27 | * This collection of utility functions is primarly intended 28 | * for debugging, but could also serve a role in error reporting. 29 | * 30 | * FIXME: need to add more protocols. 31 | */ 32 | 33 | #include 34 | #include 35 | 36 | #include 37 | #include 38 | 39 | /** 40 | * dump_eth_pkt - prints an ethernet header 41 | * @loglvl: the log level to use 42 | * @hdr: the ethernet header 43 | */ 44 | void dump_eth_pkt(int loglvl, struct eth_hdr *hdr) 45 | { 46 | struct eth_addr *dmac = &hdr->dhost; 47 | struct eth_addr *smac = &hdr->shost; 48 | 49 | logk(loglvl, "ETHERNET packet dump\n"); 50 | logk(loglvl, "\tdst MAC: %02X:%02X:%02X:%02X:%02X:%02X\n", 51 | dmac->addr[0], dmac->addr[1], dmac->addr[2], 52 | dmac->addr[3], dmac->addr[4], dmac->addr[5]); 53 | logk(loglvl, "\tsrc MAC: %02X:%02X:%02X:%02X:%02X:%02X\n", 54 | smac->addr[0], smac->addr[1], smac->addr[2], 55 | smac->addr[3], smac->addr[4], smac->addr[5]); 56 | logk(loglvl, "\tframe type: %x\n", ntoh16(hdr->type)); 57 | } 58 | 59 | /** 60 | * dump_arp_pkt - prints an arp header 61 | * @loglvl: the log level to use 62 | * @arphdr: the arp header 63 | * @ethip: the arp payload (can be NULL) 64 | * 65 | * If @ethip is NULL, then assumes an unsupported htype and/or ptype. 66 | */ 67 | void dump_arp_pkt(int loglvl, 68 | struct arp_hdr *arphdr, 69 | struct arp_hdr_ethip *ethip) 70 | { 71 | struct eth_addr *smac = ðip->sender_mac; 72 | struct eth_addr *tmac = ðip->target_mac; 73 | uint16_t op; 74 | uint32_t sip, tip; 75 | 76 | op = ntoh16(arphdr->op); 77 | sip = ntoh32(ethip->sender_ip.addr); 78 | tip = ntoh32(ethip->target_ip.addr); 79 | 80 | logk(loglvl, "ARP packet dump: op %s\n", 81 | (op == ARP_OP_REQUEST) ? "request" : "response"); 82 | 83 | if (!ethip) { 84 | logk(loglvl, "\tunsupported htype %d, ptype %d\n", 85 | ntoh16(arphdr->htype), ntoh16(arphdr->ptype)); 86 | return; 87 | } 88 | 89 | logk(loglvl, "\tsender MAC:\t%02X:%02X:%02X:%02X:%02X:%02X\n", 90 | smac->addr[0], smac->addr[1], smac->addr[2], 91 | smac->addr[3], smac->addr[4], smac->addr[5]); 92 | logk(loglvl, "\tsender IP:\t%d.%d.%d.%d\n", 93 | ((sip >> 24) & 0xff), ((sip >> 16) & 0xff), 94 | ((sip >> 8) & 0xff), (sip & 0xff)); 95 | logk(loglvl, "\ttarget MAC:\t%02X:%02X:%02X:%02X:%02X:%02X\n", 96 | tmac->addr[0], tmac->addr[1], tmac->addr[2], 97 | tmac->addr[3], tmac->addr[4], tmac->addr[5]); 98 | logk(loglvl, "\ttarget IP:\t%d.%d.%d.%d\n", 99 | ((tip >> 24) & 0xff), ((tip >> 16) & 0xff), 100 | ((tip >> 8) & 0xff), (tip & 0xff)); 101 | } 102 | 103 | -------------------------------------------------------------------------------- /dp/net/icmp.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * icmp.c - Internet Control Message Protocol support 26 | * 27 | * See RFC 792 for more details. 28 | */ 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | 36 | #include 37 | 38 | #include 39 | #include 40 | #include 41 | 42 | #include "net.h" 43 | 44 | static int icmp_reflect(struct eth_fg *cur_fg, struct mbuf *pkt, struct icmp_hdr *hdr, int len) 45 | { 46 | struct eth_hdr *ethhdr = mbuf_mtod(pkt, struct eth_hdr *); 47 | struct ip_hdr *iphdr = mbuf_nextd(ethhdr, struct ip_hdr *); 48 | int ret; 49 | 50 | ethhdr->dhost = ethhdr->shost; 51 | ethhdr->shost = CFG.mac; 52 | 53 | /* FIXME: check for invalid (e.g. multicast) src addr */ 54 | iphdr->dst_addr = iphdr->src_addr; 55 | iphdr->src_addr.addr = hton32(CFG.host_addr.addr); 56 | 57 | hdr->chksum = 0; 58 | hdr->chksum = chksum_internet((void *) hdr, len); 59 | 60 | pkt->ol_flags = 0; 61 | 62 | 63 | ret = eth_send_one(percpu_get(eth_txqs)[cur_fg->dev_idx], pkt, pkt->len); 64 | 65 | if (unlikely(ret)) { 66 | mbuf_free(pkt); 67 | return -EIO; 68 | } 69 | 70 | return 0; 71 | } 72 | 73 | /* 74 | * icmp_input - handles an input ICMP packet 75 | * @pkt: the packet 76 | * @hdr: the ICMP header 77 | */ 78 | void icmp_input(struct eth_fg *cur_fg, struct mbuf *pkt, struct icmp_hdr *hdr, int len) 79 | { 80 | if (len < ICMP_MINLEN) 81 | goto out; 82 | if (chksum_internet((void *) hdr, len)) 83 | goto out; 84 | 85 | log_debug("icmp: got request type %d, code %d\n", 86 | hdr->type, hdr->code); 87 | 88 | switch (hdr->type) { 89 | case ICMP_ECHO: 90 | hdr->type = ICMP_ECHOREPLY; 91 | icmp_reflect(cur_fg, pkt, hdr, len); 92 | break; 93 | case ICMP_ECHOREPLY: { 94 | uint16_t *seq; 95 | uint64_t *icmptimestamp; 96 | uint64_t time; 97 | 98 | seq = mbuf_nextd_off(hdr, uint16_t *, sizeof(struct icmp_hdr) + 2); 99 | icmptimestamp = mbuf_nextd_off(hdr, uint64_t *, sizeof(struct icmp_hdr) + 4); 100 | 101 | time = (rdtsc() - *icmptimestamp) / cycles_per_us; 102 | 103 | log_info("icmp: echo reply: %d bytes: icmp_req=%d time=%lld us\n", 104 | len, ntoh16(*seq), time); 105 | goto out; 106 | } 107 | default: 108 | goto out; 109 | } 110 | 111 | return; 112 | 113 | out: 114 | mbuf_free(pkt); 115 | } 116 | 117 | int icmp_echo(struct eth_fg *cur_fg, struct ip_addr *dest, uint16_t id, uint16_t seq, uint64_t timestamp) 118 | { 119 | int ret; 120 | struct mbuf *pkt; 121 | struct eth_hdr *ethhdr; 122 | struct ip_hdr *iphdr; 123 | struct icmp_pkt *icmppkt; 124 | uint64_t *icmptimestamp; 125 | uint16_t len; 126 | 127 | pkt = mbuf_alloc_local(); 128 | if (unlikely(!pkt)) 129 | return -ENOMEM; 130 | 131 | ethhdr = mbuf_mtod(pkt, struct eth_hdr *); 132 | iphdr = mbuf_nextd(ethhdr, struct ip_hdr *); 133 | icmppkt = mbuf_nextd(iphdr, struct icmp_pkt *); 134 | icmptimestamp = mbuf_nextd_off(icmppkt, uint64_t *, sizeof(struct icmp_hdr) + 4); 135 | 136 | len = sizeof(struct icmp_hdr) + 4 + sizeof(uint64_t); 137 | 138 | iphdr->header_len = sizeof(struct ip_hdr) / 4; 139 | iphdr->version = 4; 140 | iphdr->tos = 0; 141 | iphdr->len = hton16(sizeof(struct ip_hdr) + len); 142 | iphdr->id = 0; 143 | iphdr->off = 0; 144 | iphdr->ttl = 64; 145 | iphdr->chksum = 0; 146 | iphdr->proto = IPPROTO_ICMP; 147 | iphdr->src_addr.addr = hton32(CFG.host_addr.addr); 148 | iphdr->dst_addr.addr = hton32(dest->addr); 149 | iphdr->chksum = chksum_internet((void *) iphdr, sizeof(struct ip_hdr)); 150 | 151 | icmppkt->hdr.type = ICMP_ECHO; 152 | icmppkt->hdr.code = ICMP_ECHOREPLY; 153 | icmppkt->hdr.chksum = 0; 154 | icmppkt->icmp_id = hton16(id); 155 | icmppkt->icmp_seq = hton16(seq); 156 | *icmptimestamp = timestamp; 157 | icmppkt->hdr.chksum = chksum_internet((void *) icmppkt, len); 158 | 159 | pkt->ol_flags = 0; 160 | 161 | /* FIXME -- unclear if fg is/should be set */ 162 | ret = ip_send_one(cur_fg, dest, pkt, sizeof(struct eth_hdr) + sizeof(struct ip_hdr) + len); 163 | 164 | if (unlikely(ret)) { 165 | mbuf_free(pkt); 166 | return -EIO; 167 | } 168 | 169 | return 0; 170 | } 171 | -------------------------------------------------------------------------------- /dp/net/net.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * net.c - the main file for the network subsystem 26 | */ 27 | 28 | #include 29 | #include 30 | #include 31 | 32 | #include "net.h" 33 | 34 | static void net_dump_cfg(void) 35 | { 36 | char str[IP_ADDR_STR_LEN]; 37 | struct ip_addr mask = {CFG.mask}; 38 | 39 | log_info("net: using the following configuration:\n"); 40 | 41 | ip_addr_to_str((struct ip_addr *)&CFG.host_addr, str); 42 | log_info("\thost IP:\t%s\n", str); 43 | ip_addr_to_str((struct ip_addr *)&CFG.broadcast_addr, str); 44 | log_info("\tbroadcast IP:\t%s\n", str); 45 | ip_addr_to_str((struct ip_addr *)&CFG.gateway_addr, str); 46 | log_info("\tgateway IP:\t%s\n", str); 47 | ip_addr_to_str(&mask, str); 48 | log_info("\tsubnet mask:\t%s\n", str); 49 | } 50 | 51 | /** 52 | * net_init - initializes the network stack 53 | * 54 | * Returns 0 if successful, otherwise fail. 55 | */ 56 | int net_init(void) 57 | { 58 | int ret; 59 | 60 | ret = arp_init(); 61 | if (ret) { 62 | log_err("net: failed to initialize arp\n"); 63 | return ret; 64 | } 65 | 66 | return 0; 67 | } 68 | 69 | /** 70 | * net_cfg - load the network configuration parameters 71 | * 72 | * Returns 0 if successful, otherwise fail. 73 | */ 74 | int net_cfg(void) 75 | { 76 | net_dump_cfg(); 77 | 78 | return 0; 79 | } 80 | 81 | -------------------------------------------------------------------------------- /dp/net/net.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * net.h - the network stack local header 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | #include 32 | #include 33 | 34 | #include 35 | #include 36 | #include 37 | 38 | /* Address Resolution Protocol (ARP) definitions */ 39 | extern int arp_lookup_mac(struct ip_addr *addr, struct eth_addr *mac); 40 | extern int arp_insert(struct ip_addr *addr, struct eth_addr *mac); 41 | extern void arp_input(struct mbuf *pkt, struct arp_hdr *hdr); 42 | extern int arp_init(void); 43 | 44 | /* Internet Control Message Protocol (ICMP) definitions */ 45 | extern void icmp_input(struct eth_fg *, struct mbuf *pkt, struct icmp_hdr *hdr, int len); 46 | 47 | /* Unreliable Datagram Protocol (UDP) definitions */ 48 | extern void udp_input(struct mbuf *pkt, struct ip_hdr *iphdr, 49 | struct udp_hdr *udphdr); 50 | 51 | /* Transmission Control Protocol (TCP) definitions */ 52 | /* FIXME: change when we integrate better with LWIP */ 53 | extern void tcp_input_tmp(struct eth_fg *, struct mbuf *pkt, struct ip_hdr *iphdr, void *tcphdr); 54 | extern int tcp_api_init(void); 55 | extern int tcp_api_init_fg(void); 56 | 57 | /** 58 | * ip_setup_header - outputs a typical IP header 59 | * @iphdr: a pointer to the header 60 | * @proto: the protocol 61 | * @saddr: the source address 62 | * @daddr: the destination address 63 | * @l4len: the length of the L4 (e.g. UDP or TCP) header and data. 64 | */ 65 | static inline void ip_setup_header(struct ip_hdr *iphdr, uint8_t proto, 66 | uint32_t saddr, uint32_t daddr, 67 | uint16_t l4len) 68 | { 69 | iphdr->header_len = sizeof(struct ip_hdr) / 4; 70 | iphdr->version = 4; 71 | iphdr->tos = 0; 72 | iphdr->len = hton16(sizeof(struct ip_hdr) + l4len); 73 | iphdr->id = 0; 74 | iphdr->off = 0; 75 | iphdr->ttl = 64; 76 | iphdr->proto = proto; 77 | iphdr->chksum = 0; 78 | iphdr->src_addr.addr = hton32(saddr); 79 | iphdr->dst_addr.addr = hton32(daddr); 80 | } 81 | 82 | int ip_send_one(struct eth_fg *cur_fg, struct ip_addr *dst_addr, struct mbuf *pkt, size_t len); 83 | int arp_add_pending_pkt(struct ip_addr *dst_addr, struct eth_fg *fg, struct mbuf *mbuf, size_t len); 84 | -------------------------------------------------------------------------------- /dp/sandbox/dir.mk: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | SRC = main.c umm.c trap.c exec.c 23 | $(eval $(call register_dir, sandbox, $(SRC))) 24 | -------------------------------------------------------------------------------- /dp/sandbox/sandbox.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * sandbox.h - the main local header 26 | */ 27 | 28 | #ifndef __DUNESB_SANDBOX_H__ 29 | #define __DUNESB_SANDBOX_H__ 30 | 31 | #include 32 | #include 33 | #include 34 | 35 | #include 36 | 37 | #include 38 | 39 | #define LOADER_VADDR_OFF 0x6F000000 40 | #define APP_STACK_SIZE 0x800000 /* 8 megabytes */ 41 | 42 | /** 43 | * mem_ref_is_safe - determines if a memory range belongs to the sandboxed app 44 | * @ptr: the base address 45 | * @len: the length 46 | */ 47 | static inline bool mem_ref_is_safe(const void *ptr, size_t len) 48 | { 49 | uintptr_t begin = (uintptr_t) ptr; 50 | uintptr_t end = (uintptr_t)(ptr + len); 51 | 52 | /* limit possible overflows */ 53 | if (len > MEM_USER_DIRECT_END_ADDR - MEM_USER_DIRECT_BASE_ADDR) 54 | return false; 55 | 56 | /* allow ELF data */ 57 | if (begin < MEM_IX_BASE_ADDR && end < MEM_IX_BASE_ADDR) 58 | return true; 59 | 60 | /* allow the user direct memory area */ 61 | if (begin >= MEM_USER_DIRECT_BASE_ADDR && 62 | end <= MEM_USER_DIRECT_END_ADDR) 63 | return true; 64 | 65 | /* default deny everything else */ 66 | return false; 67 | } 68 | 69 | extern int check_extent(const void *ptr, size_t len); 70 | extern int check_string(const void *ptr); 71 | 72 | static inline long get_err(long ret) 73 | { 74 | if (ret < 0) 75 | return -errno; 76 | else 77 | return ret; 78 | } 79 | 80 | extern int elf_load(const char *path); 81 | 82 | extern unsigned long umm_brk(unsigned long brk); 83 | extern unsigned long umm_mmap(void *addr, size_t length, int prot, int flags, 84 | int fd, off_t offset); 85 | extern int umm_munmap(void *addr, size_t len); 86 | extern int umm_mprotect(void *addr, size_t len, unsigned long prot); 87 | extern void *umm_shmat(int shmid, void *addr, int shmflg); 88 | extern int umm_alloc_stack(uintptr_t *stack_top); 89 | extern void *umm_mremap(void *old_address, size_t old_size, 90 | size_t new_size, int flags, void *new_address); 91 | 92 | extern int trap_init(void); 93 | 94 | #endif /* __DUNESB_SANDBOX_H__ */ 95 | 96 | -------------------------------------------------------------------------------- /inc/asm/chksum.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * chksum.h - utilities for calculating checksums 26 | */ 27 | 28 | #pragma once 29 | 30 | /** 31 | * chksum_internet - performs an internet checksum on a buffer 32 | * @buf: the buffer 33 | * @len: the length in bytes 34 | * 35 | * An internet checksum is a 16-bit one's complement sum. Details 36 | * are described in RFC 1071. 37 | * 38 | * Returns a 16-bit checksum value. 39 | */ 40 | static inline uint16_t chksum_internet(const char *buf, int len) 41 | { 42 | uint64_t sum; 43 | 44 | asm volatile("xorq %0, %0\n" 45 | 46 | /* process 8 byte chunks */ 47 | "movl %2, %%edx\n" 48 | "shrl $3, %%edx\n" 49 | "cmp $0, %%edx\n" 50 | "jz 2f\n" 51 | "1: adcq (%1), %0\n" 52 | "leaq 8(%1), %1\n" 53 | "decl %%edx\n" 54 | "jne 1b\n" 55 | "adcq $0, %0\n" 56 | 57 | /* process 4 byte (if left) */ 58 | "2: test $4, %2\n" 59 | "je 3f\n" 60 | "movl (%1), %%edx\n" 61 | "addq %%rdx, %0\n" 62 | "adcq $0, %0\n" 63 | "leaq 4(%1), %1\n" 64 | 65 | /* process 2 byte (if left) */ 66 | "3: test $2, %2\n" 67 | "je 4f\n" 68 | "movzxw (%1), %%rdx\n" 69 | "addq %%rdx, %0\n" 70 | "adcq $0, %0\n" 71 | "leaq 2(%1), %1\n" 72 | 73 | /* process 1 byte (if left) */ 74 | "4: test $1, %2\n" 75 | "je 5f\n" 76 | "movzxb (%1), %%rdx\n" 77 | "addq %%rdx, %0\n" 78 | "adcq $0, %0\n" 79 | 80 | /* fold into 16-bit answer */ 81 | "5: movq %0, %1\n" 82 | "shrq $32, %0\n" 83 | "addl %k1, %k0\n" 84 | "adcl $0, %k0\n" 85 | "movq %0, %1\n" 86 | "shrl $16, %k0\n" 87 | "addw %w1, %w0\n" 88 | "adcw $0, %w0\n" 89 | "not %0\n" 90 | 91 | : "=&r"(sum), "=r"(buf) 92 | : "r"(len), "1"(buf) : "%rdx", "cc", "memory"); 93 | 94 | return (uint16_t) sum; 95 | } 96 | 97 | -------------------------------------------------------------------------------- /inc/asm/cpu.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * cpu.h - definitions for x86_64 CPUs 26 | */ 27 | 28 | #pragma once 29 | 30 | /* 31 | * Endianness 32 | */ 33 | 34 | #define __LITTLE_ENDIAN 1234 35 | #define __BIG_ENDIAN 4321 36 | 37 | #define __BYTE_ORDER __LITTLE_ENDIAN 38 | 39 | 40 | /* 41 | * Word Size 42 | */ 43 | 44 | #define __32BIT_WORDS 32 45 | #define __64BIT_WORDS 64 46 | 47 | #define __WORD_SIZE __64BIT_WORDS 48 | 49 | #define CACHE_LINE_SIZE 64 50 | 51 | #define MSR_PKG_ENERGY_STATUS 0x00000611 52 | 53 | #define cpu_relax() asm volatile("pause") 54 | 55 | #define cpu_serialize() \ 56 | asm volatile("cpuid" : : : "%rax", "%rbx", "%rcx", "%rdx") 57 | 58 | static inline unsigned long rdtsc(void) 59 | { 60 | unsigned int a, d; 61 | asm volatile("rdtsc" : "=a"(a), "=d"(d)); 62 | return ((unsigned long) a) | (((unsigned long) d) << 32); 63 | } 64 | 65 | static inline unsigned long rdtscp(unsigned int *aux) 66 | { 67 | unsigned int a, d, c; 68 | asm volatile("rdtscp" : "=a"(a), "=d"(d), "=c"(c)); 69 | if (aux) 70 | *aux = c; 71 | return ((unsigned long) a) | (((unsigned long) d) << 32); 72 | } 73 | 74 | static inline unsigned long rdmsr(unsigned int msr) 75 | { 76 | unsigned low, high; 77 | 78 | asm volatile("rdmsr" : "=a"(low), "=d"(high) : "c"(msr)); 79 | return low | ((unsigned long)high << 32); 80 | } 81 | -------------------------------------------------------------------------------- /inc/asm/uaccess.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * uaccess.h - routines for safely accessing user memory 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | 32 | #define ASM_REGISTER_FIXUP(fault_addr, fix_addr) \ 33 | ".pushsection \"__fixup_tbl\",\"a\"\n" \ 34 | ".balign 16\n" \ 35 | ".quad(" #fault_addr ")\n" \ 36 | ".quad(" #fix_addr ")\n" \ 37 | ".popsection\n" 38 | 39 | #define ASM_START_FIXUP ".section .fixup,\"ax\"\n" 40 | #define ASM_END_FIXUP ".previous\n" 41 | 42 | extern volatile int uaccess_fault; 43 | 44 | /** 45 | * uaccess_peekq - safely read a 64-bit word of memory 46 | * @addr: the address 47 | * 48 | * Returns the value. 49 | */ 50 | static inline uint64_t uaccess_peekq(const uint64_t *addr) 51 | { 52 | uint64_t ret; 53 | 54 | asm volatile("1: movq (%2), %0\n" 55 | "2:\n" 56 | ASM_START_FIXUP 57 | "3: movl $1, %1\n" 58 | "jmp 2b\n" 59 | ASM_END_FIXUP 60 | ASM_REGISTER_FIXUP(1b, 3b) : 61 | "=r"(ret), "=m"(uaccess_fault) : 62 | "r"(addr) : "memory", "cc"); 63 | 64 | return ret; 65 | } 66 | 67 | /** 68 | * uaccess_pokeq - safely writes a 64-bit word of memory 69 | * @addr: the address 70 | * @val: the value to write 71 | */ 72 | static inline void uaccess_pokeq(uint64_t *addr, uint64_t val) 73 | { 74 | asm volatile("1: movq %1, (%2)\n" 75 | "2:\n" 76 | ASM_START_FIXUP 77 | "3: movl $1, %0\n" 78 | "jmp 2b\n" 79 | ASM_END_FIXUP 80 | ASM_REGISTER_FIXUP(1b, 3b) : 81 | "=m"(uaccess_fault) : 82 | "r"(val), "r"(addr) : "memory", "cc"); 83 | } 84 | 85 | /** 86 | * uaccess_check_fault - determines if a peek or poke caused a fault 87 | * 88 | * Returns true if there was a fault, otherwise false. 89 | */ 90 | static inline bool uaccess_check_fault(void) 91 | { 92 | if (uaccess_fault) { 93 | uaccess_fault = 0; 94 | return true; 95 | } 96 | 97 | return false; 98 | } 99 | 100 | /** 101 | * uaccess_copy_user - copies memory from or to the user safely 102 | * @src: the source address 103 | * @dst: the destination address 104 | * @len: the number of bytes to copy 105 | * 106 | * Returns 0 if successful, otherwise -EFAULT if there was a bad address. 107 | */ 108 | static inline int uaccess_copy_user(const char *src, char *dst, int len) 109 | { 110 | int ret; 111 | 112 | asm volatile("1: rep\n" 113 | "movsb\n" 114 | "xorl %0, %0\n" 115 | "2:\n" 116 | ASM_START_FIXUP 117 | "3:movl %c[errno], %0\n" 118 | "jmp 2b\n" 119 | ASM_END_FIXUP 120 | ASM_REGISTER_FIXUP(1b, 3b) : 121 | "=r"(ret), "+S"(src), "+D"(dst), "+c"(len) : 122 | [errno]"i"(-EFAULT) : "memory"); 123 | 124 | return ret; 125 | } 126 | 127 | -------------------------------------------------------------------------------- /inc/ix/atomic.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * atomic.h - utilities for atomically manipulating memory 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | 32 | /** 33 | * mb - a memory barrier 34 | * 35 | * Ensures all loads and stores before the barrier complete 36 | * before all loads and stores after the barrier. 37 | */ 38 | #define mb() _mm_mfence() 39 | 40 | /** 41 | * rmb - a read memory barrier 42 | * 43 | * Ensures all loads before the barrier complete before 44 | * all loads after the barrier. 45 | */ 46 | #define rmb() _mm_lfence() 47 | 48 | /** 49 | * wmb - a write memory barrier 50 | * 51 | * Ensures all stores before the barrier complete before 52 | * all stores after the barrier. 53 | */ 54 | #define wmb() _mm_sfence() 55 | 56 | #include 57 | 58 | #define ATOMIC_INIT(val) {(val)} 59 | 60 | static inline int atomic_read(const atomic_t *a) 61 | { 62 | return *((volatile int *) &a->cnt); 63 | } 64 | 65 | static inline void atomic_write(atomic_t *a, int val) 66 | { 67 | a->cnt = val; 68 | } 69 | 70 | static inline int atomic_fetch_and_add(atomic_t *a, int val) 71 | { 72 | return __sync_fetch_and_add(&a->cnt, val); 73 | } 74 | 75 | static inline int atomic_fetch_and_sub(atomic_t *a, int val) 76 | { 77 | return __sync_fetch_and_add(&a->cnt, val); 78 | } 79 | 80 | static inline int atomic_add_and_fetch(atomic_t *a, int val) 81 | { 82 | return __sync_add_and_fetch(&a->cnt, val); 83 | } 84 | 85 | static inline int atomic_sub_and_fetch(atomic_t *a, int val) 86 | { 87 | return __sync_sub_and_fetch(&a->cnt, val); 88 | } 89 | 90 | static inline void atomic_inc(atomic_t *a) 91 | { 92 | atomic_fetch_and_add(a, 1); 93 | } 94 | 95 | static inline bool atomic_dec_and_test(atomic_t *a) 96 | { 97 | return (atomic_sub_and_fetch(a, 1) == 0); 98 | } 99 | 100 | static inline bool atomic_cmpxchg(atomic_t *a, int old, int new) 101 | { 102 | return __sync_bool_compare_and_swap(&a->cnt, old, new); 103 | } 104 | 105 | static inline long atomic64_read(const atomic64_t *a) 106 | { 107 | return *((volatile long *) &a->cnt); 108 | } 109 | 110 | static inline void atomic64_write(atomic64_t *a, long val) 111 | { 112 | a->cnt = val; 113 | } 114 | 115 | static inline long atomic64_fetch_and_add(atomic64_t *a, long val) 116 | { 117 | return __sync_fetch_and_add(&a->cnt, val); 118 | } 119 | 120 | static inline long atomic64_fetch_and_sub(atomic64_t *a, long val) 121 | { 122 | return __sync_fetch_and_add(&a->cnt, val); 123 | } 124 | 125 | static inline long atomic64_add_and_fetch(atomic64_t *a, long val) 126 | { 127 | return __sync_add_and_fetch(&a->cnt, val); 128 | } 129 | 130 | static inline long atomic64_sub_and_fetch(atomic64_t *a, long val) 131 | { 132 | return __sync_sub_and_fetch(&a->cnt, val); 133 | } 134 | 135 | static inline void atomic64_inc(atomic64_t *a) 136 | { 137 | atomic64_fetch_and_add(a, 1); 138 | } 139 | 140 | static inline bool atomic64_dec_and_test(atomic64_t *a) 141 | { 142 | return (atomic64_sub_and_fetch(a, 1) == 0); 143 | } 144 | 145 | static inline bool atomic64_cmpxchg(atomic64_t *a, long old, long new) 146 | { 147 | return __sync_bool_compare_and_swap(&a->cnt, old, new); 148 | } 149 | 150 | -------------------------------------------------------------------------------- /inc/ix/bitmap.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * bitmap.h - a library for bit array manipulation 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | 32 | #include 33 | 34 | #define BITS_PER_LONG (sizeof(long) * 8) 35 | #define BITMAP_LONG_SIZE(nbits) \ 36 | div_up(nbits, BITS_PER_LONG) 37 | 38 | #define DEFINE_BITMAP(name, nbits) \ 39 | unsigned long name[BITMAP_LONG_SIZE(nbits)] 40 | 41 | typedef unsigned long *bitmap_ptr; 42 | 43 | #define BITMAP_POS_IDX(pos) ((pos) / BITS_PER_LONG) 44 | #define BITMAP_POS_SHIFT(pos) ((pos) & (BITS_PER_LONG - 1)) 45 | 46 | /** 47 | * bitmap_set - sets a bit in the bitmap 48 | * @bits: the bitmap 49 | * @pos: the bit number 50 | */ 51 | static inline void bitmap_set(unsigned long *bits, int pos) 52 | { 53 | bits[BITMAP_POS_IDX(pos)] |= (1ul << BITMAP_POS_SHIFT(pos)); 54 | } 55 | 56 | /** 57 | * bitmap_clear - clears a bit in the bitmap 58 | * @bits: the bitmap 59 | * @pos: the bit number 60 | */ 61 | static inline void bitmap_clear(unsigned long *bits, int pos) 62 | { 63 | bits[BITMAP_POS_IDX(pos)] &= ~(1ul << BITMAP_POS_SHIFT(pos)); 64 | } 65 | 66 | /** 67 | * bitmap_test - tests if a bit is set in the bitmap 68 | * @bits: the bitmap 69 | * @pos: the bit number 70 | * 71 | * Returns true if the bit is set, otherwise false. 72 | */ 73 | static inline bool bitmap_test(unsigned long *bits, int pos) 74 | { 75 | return (bits[BITMAP_POS_IDX(pos)] & (1ul << BITMAP_POS_SHIFT(pos))) != 0; 76 | } 77 | 78 | /** 79 | * bitmap_init - initializes a bitmap 80 | * @bits: the bitmap 81 | * @nbits: the number of total bits 82 | * @state: if true, all bits are set, otherwise all bits are cleared 83 | */ 84 | static inline void bitmap_init(unsigned long *bits, int nbits, bool state) 85 | { 86 | memset(bits, state ? 0xff : 0x00, BITMAP_LONG_SIZE(nbits) * sizeof(long)); 87 | } 88 | 89 | -------------------------------------------------------------------------------- /inc/ix/byteorder.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * byteorder.h - utilties for swapping bytes and converting endianness 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | #include 32 | 33 | static inline uint16_t __bswap16(uint16_t val) 34 | { 35 | #ifdef HAS_BUILTIN_BSWAP 36 | return __builtin_bswap16(val); 37 | #else 38 | return (((val & 0x00ffU) << 8) | 39 | ((val & 0xff00U) >> 8)); 40 | #endif 41 | } 42 | 43 | static inline uint32_t __bswap32(uint32_t val) 44 | { 45 | #ifdef HAS_BUILTIN_BSWAP 46 | return __builtin_bswap32(val); 47 | #else 48 | return (((val & 0x000000ffUL) << 24) | 49 | ((val & 0x0000ff00UL) << 8) | 50 | ((val & 0x00ff0000UL) >> 8) | 51 | ((val & 0xff000000UL) >> 24)); 52 | #endif 53 | } 54 | 55 | static inline uint64_t __bswap64(uint64_t val) 56 | { 57 | #ifdef HAS_BUILTIN_BSWAP 58 | return __builtin_bswap64(val); 59 | #else 60 | return (((val & 0x00000000000000ffULL) << 56) | 61 | ((val & 0x000000000000ff00ULL) << 40) | 62 | ((val & 0x0000000000ff0000ULL) << 24) | 63 | ((val & 0x00000000ff000000ULL) << 8) | 64 | ((val & 0x000000ff00000000ULL) >> 8) | 65 | ((val & 0x0000ff0000000000ULL) >> 24) | 66 | ((val & 0x00ff000000000000ULL) >> 40) | 67 | ((val & 0xff00000000000000ULL) >> 56)); 68 | #endif 69 | } 70 | 71 | #ifndef __BYTE_ORDER 72 | #error __BYTE_ORDER is undefined 73 | #endif 74 | 75 | #if __BYTE_ORDER == __LITTLE_ENDIAN 76 | 77 | #define cpu_to_le16(x) (x) 78 | #define cpu_to_le32(x) (x) 79 | #define cpu_to_le64(x) (x) 80 | #define cpu_to_be16(x) (__bswap16(x)) 81 | #define cpu_to_be32(x) (__bswap32(x)) 82 | #define cpu_to_be64(x) (__bswap64(x)) 83 | 84 | #define le16_to_cpu(x) (x) 85 | #define le32_to_cpu(x) (x) 86 | #define le64_to_cpu(x) (x) 87 | #define be16_to_cpu(x) (__bswap16(x)) 88 | #define be32_to_cpu(x) (__bswap32(x)) 89 | #define be64_to_cpu(x) (__bswap64(x)) 90 | 91 | #else /* __BYTE_ORDER == __LITLE_ENDIAN */ 92 | 93 | #define cpu_to_le16(x) (__bswap16(x)) 94 | #define cpu_to_le32(x) (__bswap32(x)) 95 | #define cpu_to_le64(x) (__bswap64(x)) 96 | #define cpu_to_be16(x) (x) 97 | #define cpu_to_be32(x) (x) 98 | #define cpu_to_be64(x) (x) 99 | 100 | #define le16_to_cpu(x) (__bswap16(x)) 101 | #define le32_to_cpu(x) (__bswap32(x)) 102 | #define le64_to_cpu(x) (__bswap64(x)) 103 | #define be16_to_cpu(x) (x) 104 | #define be32_to_cpu(x) (x) 105 | #define be64_to_cpu(x) (x) 106 | 107 | #endif /* __BYTE_ORDER == __LITTLE_ENDIAN */ 108 | 109 | #define ntoh16(x) (be16_to_cpu(x)) 110 | #define ntoh32(x) (be32_to_cpu(x)) 111 | #define ntoh64(x) (be64_to_cpu(x)) 112 | 113 | #define hton16(x) (cpu_to_be16(x)) 114 | #define hton32(x) (cpu_to_be32(x)) 115 | #define hton64(x) (cpu_to_be64(x)) 116 | 117 | -------------------------------------------------------------------------------- /inc/ix/cfg.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * cfg.h - configuration parameters 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | #include 32 | 33 | 34 | #define CFG_MAX_PORTS 16 35 | #define CFG_MAX_CPU 128 36 | #define CFG_MAX_ETHDEV 16 37 | 38 | 39 | struct cfg_ip_addr { 40 | uint32_t addr; 41 | }; 42 | 43 | struct cfg_parameters { 44 | struct cfg_ip_addr host_addr; 45 | struct cfg_ip_addr broadcast_addr; 46 | struct cfg_ip_addr gateway_addr; 47 | uint32_t mask; 48 | 49 | struct eth_addr mac; 50 | 51 | int num_cpus; 52 | unsigned int cpu[CFG_MAX_CPU]; 53 | 54 | int num_ethdev; 55 | struct pci_addr ethdev[CFG_MAX_ETHDEV]; 56 | 57 | int num_ports; 58 | uint16_t ports[CFG_MAX_PORTS]; 59 | 60 | char loader_path[256]; 61 | }; 62 | 63 | extern struct cfg_parameters CFG; 64 | 65 | 66 | 67 | 68 | extern int cfg_init(int argc, char *argv[], int *args_parsed); 69 | 70 | -------------------------------------------------------------------------------- /inc/ix/compiler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * compiler.h - useful compiler hints, intrinsics, and attributes 26 | */ 27 | 28 | #pragma once 29 | 30 | #define likely(x) __builtin_expect(!!(x), 1) 31 | #define unlikely(x) __builtin_expect(!!(x), 0) 32 | #define unreachable() __builtin_unreachable() 33 | 34 | #define prefetch0(x) __builtin_prefetch((x), 0, 3) 35 | #define prefetch1(x) __builtin_prefetch((x), 0, 2) 36 | #define prefetch2(x) __builtin_prefetch((x), 0, 1) 37 | #define prefetchnta(x) __builtin_prefetch((x), 0, 0) 38 | #define prefetch() prefetch0() 39 | 40 | #define clz64(x) __builtin_clzll(x) 41 | 42 | #define __packed __attribute__((packed)) 43 | #define __notused __attribute__((unused)) 44 | #define __aligned(x) __attribute__((aligned(x))) 45 | 46 | #define GCC_VERSION (__GNUC__ * 10000 \ 47 | + __GNUC_MINOR__ * 100 \ 48 | + __GNUC_PATCHLEVEL__) 49 | 50 | #if GCC_VERSION >= 40800 51 | #define HAS_BUILTIN_BSWAP 1 52 | #endif 53 | 54 | -------------------------------------------------------------------------------- /inc/ix/config.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #define CONFIG_PRINT_CONNECTION_COUNT 0 4 | 5 | #define CONFIG_STATS 0 6 | -------------------------------------------------------------------------------- /inc/ix/control_plane.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * control_plane.h - control plane definitions 26 | */ 27 | 28 | #include 29 | #include 30 | 31 | #define IDLE_FIFO_SIZE 256 32 | 33 | struct cpu_metrics { 34 | double queuing_delay; 35 | double batch_size; 36 | double queue_size[3]; 37 | long loop_duration; 38 | double idle[3]; 39 | } __aligned(64); 40 | 41 | struct flow_group_metrics { 42 | int cpu; 43 | } __aligned(64); 44 | 45 | enum cpu_state { 46 | CP_CPU_STATE_IDLE = 0, 47 | CP_CPU_STATE_RUNNING, 48 | }; 49 | 50 | enum commands { 51 | CP_CMD_NOP = 0, 52 | CP_CMD_MIGRATE, 53 | CP_CMD_IDLE, 54 | }; 55 | 56 | enum status { 57 | CP_STATUS_READY = 0, 58 | CP_STATUS_RUNNING, 59 | }; 60 | 61 | struct command_struct { 62 | enum cpu_state cpu_state; 63 | enum commands cmd_id; 64 | enum status status; 65 | union { 66 | struct { 67 | DEFINE_BITMAP(fg_bitmap, ETH_MAX_TOTAL_FG); 68 | int cpu; 69 | } migrate; 70 | struct { 71 | char fifo[IDLE_FIFO_SIZE]; 72 | } idle; 73 | }; 74 | char no_idle; 75 | }; 76 | 77 | extern volatile struct cp_shmem { 78 | uint32_t nr_flow_groups; 79 | uint32_t nr_cpus; 80 | float pkg_power; 81 | int cpu[NCPU]; 82 | struct cpu_metrics cpu_metrics[NCPU]; 83 | struct flow_group_metrics flow_group[ETH_MAX_TOTAL_FG]; 84 | struct command_struct command[NCPU]; 85 | uint32_t cycles_per_us; 86 | uint32_t scratchpad_idx; 87 | struct { 88 | long remote_queue_pkts_begin; 89 | long remote_queue_pkts_end; 90 | long local_queue_pkts; 91 | long backlog_before; 92 | long backlog_after; 93 | long timers; 94 | long timer_fired; 95 | long ts_migration_start; 96 | long ts_data_structures_done; 97 | long ts_before_backlog; 98 | long ts_after_backlog; 99 | long ts_migration_end; 100 | long ts_first_pkt_at_prev; 101 | long ts_last_pkt_at_prev; 102 | long ts_first_pkt_at_target; 103 | long ts_last_pkt_at_target; 104 | } scratchpad[1024]; 105 | } *cp_shmem; 106 | 107 | #define SCRATCHPAD (&cp_shmem->scratchpad[cp_shmem->scratchpad_idx]) 108 | #define SCRATCHPAD_NEXT do { assert(++cp_shmem->scratchpad_idx < 1024); } while (0) 109 | 110 | DECLARE_PERCPU(volatile struct command_struct *, cp_cmd); 111 | DECLARE_PERCPU(unsigned long, idle_cycles); 112 | 113 | void cp_idle(void); 114 | 115 | static inline double ema_update(double prv_value, double value, double alpha) 116 | { 117 | return alpha * value + (1 - alpha) * prv_value; 118 | } 119 | 120 | #define EMA_UPDATE(ema, value, alpha) ema = ema_update(ema, value, alpha) 121 | 122 | extern double energy_unit; 123 | -------------------------------------------------------------------------------- /inc/ix/cpu.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * cpu.h - support for multicore and percpu data. 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | #include 32 | 33 | #define NCPU 128 34 | extern int cpu_count; /* the number of available CPUs */ 35 | extern int cpus_active; /* the number of in-use CPUs */ 36 | 37 | /* used to define percpu variables */ 38 | #define DEFINE_PERCPU(type, name) \ 39 | typeof(type) name __attribute__((section(".percpu,\"\",@nobits#"))) 40 | 41 | /* used to make percpu variables externally available */ 42 | #define DECLARE_PERCPU(type, name) \ 43 | extern DEFINE_PERCPU(type, name) 44 | 45 | extern void *percpu_offsets[NCPU]; 46 | 47 | /** 48 | * percpu_get_remote - get a percpu variable on a specific core 49 | * @var: the percpu variable 50 | * @cpu: the cpu core number 51 | * 52 | * Returns a percpu variable. 53 | */ 54 | #define percpu_get_remote(var, cpu) \ 55 | (*((typeof(var) *) ((uintptr_t) &var + \ 56 | (uintptr_t) percpu_offsets[(cpu)]))) 57 | 58 | static inline void *__percpu_get(void *key) 59 | { 60 | void *offset; 61 | 62 | asm("mov %%gs:0, %0" : "=r"(offset)); 63 | 64 | return (void *)((uintptr_t) key + (uintptr_t) offset); 65 | } 66 | 67 | /** 68 | * percpu_get_addr - get the local percpu variable's address 69 | * @var: the percpu variable 70 | * 71 | * Returns a percpu variable address. 72 | */ 73 | #define percpu_get_addr(var) \ 74 | ((typeof(var) *) (__percpu_get(&var))) 75 | 76 | /** 77 | * percpu_get - get the local percpu variable 78 | * @var: the percpu variable 79 | * 80 | * Returns a percpu variable. 81 | */ 82 | #define percpu_get(var) \ 83 | (*percpu_get_addr(var)) 84 | 85 | /** 86 | * cpu_is_active - is the CPU being used? 87 | * @cpu: the cpu number 88 | * 89 | * Returns true if yes, false if no. 90 | */ 91 | #define cpu_is_active(cpu) \ 92 | (percpu_offsets[(cpu)] != NULL) 93 | 94 | static inline unsigned int __cpu_next_active(unsigned int cpu) 95 | { 96 | while (cpu < cpu_count) { 97 | cpu++; 98 | 99 | if (cpu_is_active(cpu)) 100 | return cpu; 101 | } 102 | 103 | return cpu; 104 | } 105 | 106 | /** 107 | * for_each_active_cpu - iterates over each active (used by IX) CPU 108 | * @cpu: an integer to store the cpu 109 | */ 110 | #define for_each_active_cpu(cpu) \ 111 | for ((cpu) = -1; (cpu) = __cpu_next_active(cpu); (cpu) < cpu_count) 112 | 113 | DECLARE_PERCPU(unsigned int, cpu_numa_node); 114 | DECLARE_PERCPU(unsigned int, cpu_id); 115 | DECLARE_PERCPU(unsigned int, cpu_nr); 116 | 117 | extern void cpu_do_bookkeeping(void); 118 | 119 | typedef void (*cpu_func_t)(void *data); 120 | extern int cpu_run_on_one(cpu_func_t func, void *data, 121 | unsigned int cpu); 122 | 123 | extern int cpu_init_one(unsigned int cpu); 124 | extern int cpu_init(void); 125 | -------------------------------------------------------------------------------- /inc/ix/dpdk.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | extern struct rte_mempool *dpdk_pool; 25 | 26 | uint8_t rte_eth_dev_find_free_port(void); 27 | -------------------------------------------------------------------------------- /inc/ix/drivers.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | 28 | /* driver-independent initialization function */ 29 | int driver_init(struct pci_dev *dev, struct ix_rte_eth_dev **eth); 30 | 31 | /* driver-independent configuration */ 32 | extern struct rte_eth_rxconf rx_conf; 33 | extern struct rte_eth_txconf tx_conf; 34 | 35 | /* driver specific initialization functions */ 36 | int ixgbe_init(struct ix_rte_eth_dev *dev, const char *driver_name); 37 | int i40e_init(struct ix_rte_eth_dev *dev, const char *driver_name); 38 | 39 | /* driver-independent eth_dev_ops */ 40 | void generic_allmulticast_enable(struct ix_rte_eth_dev *dev); 41 | void generic_dev_infos_get(struct ix_rte_eth_dev *dev, struct ix_rte_eth_dev_info *dev_info); 42 | int generic_link_update(struct ix_rte_eth_dev *dev, int wait_to_complete); 43 | void generic_promiscuous_disable(struct ix_rte_eth_dev *dev); 44 | int generic_fdir_add_perfect_filter(struct ix_rte_eth_dev *dev, struct rte_fdir_filter *fdir_ftr, uint16_t soft_id, uint8_t rx_queue, uint8_t drop); 45 | int generic_fdir_remove_perfect_filter(struct ix_rte_eth_dev *dev, struct rte_fdir_filter *fdir_ftr, uint16_t soft_id); 46 | int generic_rss_hash_conf_get(struct ix_rte_eth_dev *dev, struct ix_rte_eth_rss_conf *ix_reta_conf); 47 | void generic_mac_addr_add(struct ix_rte_eth_dev *dev, struct eth_addr *mac_addr, uint32_t index, uint32_t vmdq); 48 | -------------------------------------------------------------------------------- /inc/ix/kstats.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | 25 | #pragma once 26 | 27 | #include 28 | 29 | #include 30 | #include 31 | #include 32 | 33 | typedef struct kstats_distr { 34 | uint64_t count; 35 | uint64_t min_occ; 36 | uint64_t max_occ; 37 | uint64_t tot_occ; 38 | uint64_t min_lat; 39 | uint64_t max_lat; 40 | uint64_t tot_lat; 41 | 42 | } kstats_distr; 43 | 44 | 45 | typedef struct kstats_accumulate { 46 | kstats_distr *cur; 47 | uint64_t start_lat; 48 | uint64_t start_occ; 49 | uint64_t accum_time; 50 | } kstats_accumulate; 51 | 52 | 53 | typedef struct kstats { 54 | #define DEF_KSTATS(_c) kstats_distr _c 55 | #include "kstatvectors.h" 56 | } kstats; 57 | 58 | #ifdef ENABLE_KSTATS 59 | 60 | DECLARE_PERCPU(kstats, _kstats); 61 | DECLARE_PERCPU(kstats_accumulate, _kstats_accumulate); 62 | DECLARE_PERCPU(int, _kstats_packets); 63 | DECLARE_PERCPU(int, _kstats_batch_histogram[]); 64 | DECLARE_PERCPU(int, _kstats_backlog_histogram[]); 65 | 66 | #define KSTATS_BATCH_HISTOGRAM_SIZE 512 67 | #define KSTATS_BACKLOG_HISTOGRAM_SIZE 512 68 | 69 | extern void kstats_enter(kstats_distr *n, kstats_accumulate *saved_accu); 70 | extern void kstats_leave(kstats_accumulate *saved_accu); 71 | 72 | static inline void kstats_vector(kstats_distr *n) 73 | { 74 | percpu_get(_kstats_accumulate).cur = n; 75 | } 76 | 77 | static inline void kstats_packets_inc(int count) 78 | { 79 | percpu_get(_kstats_packets) += count; 80 | } 81 | 82 | static inline void kstats_batch_inc(int count) 83 | { 84 | if (count >= KSTATS_BATCH_HISTOGRAM_SIZE) 85 | panic("kstats batch histogram overflow\n"); 86 | 87 | percpu_get(_kstats_batch_histogram)[count]++; 88 | } 89 | 90 | static inline void kstats_backlog_inc(int count) 91 | { 92 | if (count >= KSTATS_BACKLOG_HISTOGRAM_SIZE) 93 | panic("kstats backlog histogram overflow\n"); 94 | 95 | percpu_get(_kstats_backlog_histogram)[count]++; 96 | } 97 | 98 | #define KSTATS_PUSH(TYPE, _save) \ 99 | kstats_enter(&(percpu_get(_kstats)).TYPE, _save) 100 | #define KSTATS_VECTOR(TYPE) \ 101 | kstats_vector(&(percpu_get(_kstats)).TYPE) 102 | #define KSTATS_POP(_save) \ 103 | kstats_leave(_save) 104 | //#define KSTATS_CURRENT_IS(TYPE) (percpu_get(_kstats_accumulate).cur==&kstatsCounters.TYPE) 105 | #define KSTATS_PACKETS_INC(_count) \ 106 | kstats_packets_inc(_count) 107 | #define KSTATS_BATCH_INC(_count) \ 108 | kstats_batch_inc(_count) 109 | #define KSTATS_BACKLOG_INC(_count) \ 110 | kstats_backlog_inc(_count) 111 | 112 | extern int kstats_init_cpu(void); 113 | 114 | #else /* ENABLE_KSTATS */ 115 | 116 | #define KSTATS_PUSH(TYPE, _save) 117 | #define KSTATS_VECTOR(TYPE) 118 | #define KSTATS_POP(_save) 119 | #define KSTATS_CURRENT_IS(TYPE) 0 120 | #define KSTATS_PACKETS_INC(_count) 121 | #define KSTATS_BATCH_INC(_count) 122 | #define KSTATS_BACKLOG_INC(_count) 123 | 124 | 125 | #endif /* ENABLE_KSTATS */ 126 | 127 | -------------------------------------------------------------------------------- /inc/ix/kstatvectors.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | 25 | /* 26 | * actually not a pragma once file 27 | */ 28 | 29 | 30 | DEF_KSTATS(none); 31 | DEF_KSTATS(idle); 32 | DEF_KSTATS(user); 33 | DEF_KSTATS(timer); 34 | DEF_KSTATS(timer_collapse); 35 | DEF_KSTATS(print_kstats); 36 | DEF_KSTATS(percpu_bookkeeping); 37 | DEF_KSTATS(tx_reclaim); 38 | DEF_KSTATS(tx_send); 39 | DEF_KSTATS(rx_poll); 40 | DEF_KSTATS(rx_recv); 41 | DEF_KSTATS(bsys); 42 | DEF_KSTATS(timer_tcp_fasttmr); 43 | DEF_KSTATS(timer_tcp_slowtmr); 44 | DEF_KSTATS(eth_input); 45 | DEF_KSTATS(tcp_input_fast_path); 46 | DEF_KSTATS(tcp_input_listen); 47 | DEF_KSTATS(tcp_output_syn); 48 | DEF_KSTATS(tcp_unified_handler); 49 | DEF_KSTATS(timer_tcp_send_delayed_ack); 50 | DEF_KSTATS(timer_handler); 51 | DEF_KSTATS(timer_tcp_retransmit); 52 | DEF_KSTATS(timer_tcp_persist); 53 | DEF_KSTATS(bsys_dispatch_one); 54 | DEF_KSTATS(bsys_tcp_accept); 55 | DEF_KSTATS(bsys_tcp_close); 56 | DEF_KSTATS(bsys_tcp_connect); 57 | DEF_KSTATS(bsys_tcp_recv_done); 58 | DEF_KSTATS(bsys_tcp_reject); 59 | DEF_KSTATS(bsys_tcp_send); 60 | DEF_KSTATS(bsys_tcp_sendv); 61 | DEF_KSTATS(bsys_udp_recv_done); 62 | DEF_KSTATS(bsys_udp_send); 63 | DEF_KSTATS(bsys_udp_sendv); 64 | 65 | DEF_KSTATS(posix_syscall); 66 | -------------------------------------------------------------------------------- /inc/ix/lock.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * lock.h - locking primitives 26 | */ 27 | 28 | #pragma once 29 | 30 | #define CONFIG_DEBUG_SPIN_LOCK 0 31 | 32 | #include 33 | #include 34 | #if __KERNEL__ && CONFIG_DEBUG_SPIN_LOCK 35 | #include 36 | #include 37 | #endif 38 | 39 | #define SPINLOCK_INITIALIZER {.locked = 0} 40 | #define DEFINE_SPINLOCK(name) \ 41 | spinlock_t name = SPINLOCK_INITIALIZER 42 | #define DECLARE_SPINLOCK(name) \ 43 | extern spinlock_t name 44 | 45 | /** 46 | * spin_lock_init - prepares a spin lock for use 47 | * @l: the spin lock 48 | */ 49 | static inline void spin_lock_init(spinlock_t *l) 50 | { 51 | l->locked = 0; 52 | } 53 | 54 | /** 55 | * spin_lock - takes a spin lock 56 | * @l: the spin lock 57 | */ 58 | static inline void spin_lock(spinlock_t *l) 59 | { 60 | #if __KERNEL__ && CONFIG_DEBUG_SPIN_LOCK 61 | long start = rdtsc(); 62 | long deadline = start + ONE_MS * cycles_per_us; 63 | #endif 64 | 65 | while (__sync_lock_test_and_set(&l->locked, 1)) { 66 | while (l->locked) { 67 | #if __KERNEL__ && CONFIG_DEBUG_SPIN_LOCK 68 | if (unlikely(rdtsc() > deadline)) { 69 | here: log_err("slow spin_lock (%ld us) at %p called from %p\n", (rdtsc() - start) / cycles_per_us, &&here, __builtin_return_address(0)); 70 | deadline += (long) ONE_SECOND * cycles_per_us; 71 | } 72 | #endif 73 | cpu_relax(); 74 | } 75 | } 76 | } 77 | 78 | /** 79 | * spin_try_lock- takes a spin lock, but only if it is available 80 | * @l: the spin lock 81 | * 82 | * Returns 1 if successful, otherwise 0 83 | */ 84 | static inline bool spin_try_lock(spinlock_t *l) 85 | { 86 | return !(__sync_lock_test_and_set(&l->locked, 1)); 87 | } 88 | 89 | /** 90 | * spin_unlock - releases a spin lock 91 | * @l: the spin lock 92 | */ 93 | static inline void spin_unlock(spinlock_t *l) 94 | { 95 | __sync_lock_release(&l->locked); 96 | } 97 | 98 | -------------------------------------------------------------------------------- /inc/ix/log.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * log.h - the logging service 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | 32 | extern __thread bool log_is_early_boot; 33 | 34 | extern void logk(int level, const char *fmt, ...); 35 | 36 | extern int max_loglevel; 37 | 38 | enum { 39 | LOG_EMERG = 0, /* system is dead */ 40 | LOG_CRIT = 1, /* critical */ 41 | LOG_ERR = 2, /* error */ 42 | LOG_WARN = 3, /* warning */ 43 | LOG_INFO = 4, /* informational */ 44 | LOG_DEBUG = 5, /* debug */ 45 | LOG_CONT = 6, /* continuation */ 46 | }; 47 | 48 | #define log_emerg(fmt, ...) logk(LOG_EMERG, fmt, ##__VA_ARGS__) 49 | #define log_crit(fmt, ...) logk(LOG_CRIT, fmt, ##__VA_ARGS__) 50 | #define log_err(fmt, ...) logk(LOG_ERR, fmt, ##__VA_ARGS__) 51 | #define log_warn(fmt, ...) logk(LOG_WARN, fmt, ##__VA_ARGS__) 52 | #define log_info(fmt, ...) logk(LOG_INFO, fmt, ##__VA_ARGS__) 53 | #define log_cont(fmt, ...) logk(LOG_CONT, fmt, ##__VA_ARGS__) 54 | 55 | #ifdef DEBUG 56 | #define log_debug(fmt, ...) logk(LOG_DEBUG, fmt, ##__VA_ARGS__) 57 | #else 58 | #define log_debug(fmt, ...) 59 | #endif 60 | 61 | #define panic(fmt, ...) \ 62 | do {logk(LOG_EMERG, fmt, ##__VA_ARGS__); exit(-1); } while (0) 63 | 64 | -------------------------------------------------------------------------------- /inc/ix/page.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * page.h - page-level memory management 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | struct page_ent { 36 | machaddr_t maddr; 37 | atomic_t refcnt; 38 | uint32_t flags; 39 | }; 40 | 41 | #define PAGE_FLAG_WILL_FREE 0x1 42 | #define PAGE_FLAG_CAN_FREE 0x2 43 | 44 | extern struct page_ent page_tbl[]; 45 | DECLARE_PERCPU(int32_t, page_refs[]); 46 | 47 | #define PAGE_NUM(addr) \ 48 | PGN_2MB((uintptr_t) (addr) - (uintptr_t) MEM_PHYS_BASE_ADDR) 49 | 50 | /** 51 | * is_page - determines an address is inside page memory 52 | * @addr: the address 53 | * 54 | * Returns true if the address is inside page memory. 55 | */ 56 | static inline bool is_page(void *addr) 57 | { 58 | return ((uintptr_t) addr >= MEM_PHYS_BASE_ADDR && 59 | (uintptr_t) addr < MEM_USER_START); 60 | } 61 | 62 | /** 63 | * is_page_region - determines if an address range is inside page memory 64 | * @addr: the base address 65 | * @len: the length of the region 66 | * 67 | * Returns true if the region is inside page memory, otherwise false. 68 | */ 69 | static inline bool is_page_region(void *addr, size_t len) 70 | { 71 | if (len > MEM_USER_START - MEM_PHYS_BASE_ADDR || 72 | (uintptr_t) addr < MEM_PHYS_BASE_ADDR || 73 | (uintptr_t) addr + len > MEM_USER_START) 74 | return false; 75 | 76 | return true; 77 | } 78 | 79 | /** 80 | * page_machaddr - gets the machine address of a page 81 | * @addr: the address of (or in) the page 82 | * 83 | * NOTE: This variant is unsafe if a reference to the page is not already held. 84 | * 85 | * Returns the machine address of the page plus the offset within the page. 86 | */ 87 | static inline machaddr_t page_machaddr(void *addr) 88 | { 89 | return page_tbl[PAGE_NUM(addr)].maddr + PGOFF_2MB(addr); 90 | } 91 | 92 | /** 93 | * page_get - pins a memory page 94 | * @addr: the address of (or in) the page 95 | * 96 | * Returns the machine address of the page plus the offset within the page. 97 | */ 98 | static inline machaddr_t page_get(void *addr) 99 | { 100 | unsigned long idx = PAGE_NUM(addr); 101 | struct page_ent *ent = &page_tbl[idx]; 102 | 103 | if (unlikely(ent->flags & PAGE_FLAG_WILL_FREE)) 104 | atomic_inc(&ent->refcnt); 105 | else 106 | percpu_get(page_refs[idx])++; 107 | 108 | return ent->maddr + PGOFF_2MB(addr); 109 | } 110 | 111 | extern void __page_put_slow(void *addr); 112 | 113 | /** 114 | * page_put - unpins an iomap memory page 115 | * @addr: the address of (or in) the page 116 | */ 117 | static inline void page_put(void *addr) 118 | { 119 | unsigned long idx = PAGE_NUM(addr); 120 | struct page_ent *ent = &page_tbl[idx]; 121 | 122 | if (unlikely(ent->flags & PAGE_FLAG_WILL_FREE)) 123 | __page_put_slow(addr); 124 | else 125 | percpu_get(page_refs[idx])--; 126 | } 127 | 128 | extern void * 129 | page_alloc_contig_on_node(unsigned int nr, int numa_node); 130 | extern void page_free(void *addr); 131 | extern void page_free_contig(void *addr, unsigned int nr); 132 | 133 | /** 134 | * page_alloc_contig - allocate contiguous pages 135 | * @nr: the number of pages to allocate 136 | * 137 | * Returns an address, or NULL if fail. 138 | */ 139 | static inline void *page_alloc_contig(unsigned int nr) 140 | { 141 | return page_alloc_contig_on_node(nr, percpu_get(cpu_numa_node)); 142 | } 143 | 144 | /** 145 | * page_alloc_on_node - allocates a page on the given numa node 146 | * @numa_node: the target numa node 147 | * 148 | * If @numa_node is -1, then any numa node can be used. 149 | * 150 | * Returns an address, or NULL if fail. 151 | */ 152 | static inline void *page_alloc_on_node(int numa_node) 153 | { 154 | return page_alloc_contig_on_node(1, numa_node); 155 | } 156 | 157 | /** 158 | * page_alloc - allocates a page 159 | * 160 | * Returns an address, or NULL if fail. 161 | */ 162 | static inline void *page_alloc(void) 163 | { 164 | return page_alloc_contig_on_node(1, percpu_get(cpu_numa_node)); 165 | } 166 | 167 | -------------------------------------------------------------------------------- /inc/ix/pci.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * pci.h - PCI bus support 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | 32 | struct pci_bar { 33 | uint64_t start; /* the start address, or zero if no resource */ 34 | uint64_t len; /* the length of the resource */ 35 | uint64_t flags; /* Linux resource flags */ 36 | }; 37 | 38 | /* NOTE: these are the same as the Linux PCI sysfs resource flags */ 39 | #define PCI_BAR_IO 0x00000100 40 | #define PCI_BAR_MEM 0x00000200 41 | #define PCI_BAR_PREFETCH 0x00002000 /* typically WC memory */ 42 | #define PCI_BAR_READONLY 0x00004000 /* typically option ROMs */ 43 | 44 | #define PCI_MAX_BARS 7 45 | 46 | struct pci_addr { 47 | uint16_t domain; 48 | uint8_t bus; 49 | uint8_t slot; 50 | uint8_t func; 51 | }; 52 | 53 | extern int pci_str_to_addr(const char *str, struct pci_addr *addr); 54 | 55 | struct pci_dev { 56 | struct pci_addr addr; 57 | 58 | uint16_t vendor_id; 59 | uint16_t device_id; 60 | uint16_t subsystem_vendor_id; 61 | uint16_t subsystem_device_id; 62 | 63 | struct pci_bar bars[PCI_MAX_BARS]; 64 | int numa_node; 65 | int max_vfs; 66 | }; 67 | 68 | extern struct pci_dev *pci_alloc_dev(const struct pci_addr *addr); 69 | extern struct pci_bar *pci_find_mem_bar(struct pci_dev *dev, int count); 70 | extern void *pci_map_mem_bar(struct pci_dev *dev, struct pci_bar *bar, bool wc); 71 | extern void pci_unmap_mem_bar(struct pci_bar *bar, void *vaddr); 72 | extern int pci_enable_device(struct pci_dev *dev); 73 | extern int pci_set_master(struct pci_dev *dev); 74 | -------------------------------------------------------------------------------- /inc/ix/perf.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | long read_perf_event(int fd); 4 | 5 | int init_perf_event(struct perf_event_attr *attr); 6 | -------------------------------------------------------------------------------- /inc/ix/profiler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | #ifdef ENABLE_PROFILER 25 | 26 | void profiler_init(void); 27 | 28 | #else /* ENABLE_PROFILER */ 29 | 30 | static inline void profiler_init(void) { } 31 | 32 | #endif /* ENABLE_PROFILER */ 33 | -------------------------------------------------------------------------------- /inc/ix/stats.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #define STATS \ 7 | COUNTER(llc_load_misses) 8 | 9 | #if CONFIG_STATS 10 | 11 | struct ix_stats { 12 | int cpus; 13 | struct ix_stats_percpu { 14 | volatile char reset; 15 | volatile char poll; 16 | #define HISTOGRAM(name, min, max, buckets) \ 17 | struct { \ 18 | long sum; \ 19 | unsigned int bucket[buckets]; \ 20 | } __attribute__((packed)) name; 21 | #define COUNTER(name) int name; 22 | STATS 23 | #undef HISTOGRAM 24 | #undef COUNTER 25 | } __attribute__((packed, aligned(64))) percpu[NCPU]; 26 | }; 27 | 28 | int stats_init(void); 29 | int stats_init_cpu(void); 30 | 31 | extern struct ix_stats *stats; 32 | 33 | static inline int stats_histogram_bucket(int value, int min, int max, int buckets) 34 | { 35 | int b; 36 | b = value - min; 37 | b = b * buckets / (max - min); 38 | if (b < 0) 39 | b = 0; 40 | else if (b > buckets - 1) 41 | b = buckets - 1; 42 | return b; 43 | } 44 | 45 | #define HISTOGRAM(name, min, max, buckets) \ 46 | static inline void stats_histogram_ ## name(int value) { \ 47 | stats->percpu[percpu_get(cpu_nr)].name.sum += value; \ 48 | stats->percpu[percpu_get(cpu_nr)].name.bucket[stats_histogram_bucket(value, min, max, buckets)]++; \ 49 | } 50 | #define COUNTER(name) \ 51 | static inline void stats_counter_ ## name(int delta) { \ 52 | stats->percpu[percpu_get(cpu_nr)].name += delta; \ 53 | } 54 | STATS 55 | #undef HISTOGRAM 56 | #undef COUNTER 57 | 58 | void stats_check_reset(void); 59 | 60 | #else 61 | 62 | #define HISTOGRAM(name, min, max, buckets) static inline void stats_histogram_ ## name(int value) { } 63 | #define COUNTER(name) static inline void stats_counter_ ## name(int delta) { } 64 | STATS 65 | #undef HISTOGRAM 66 | #undef COUNTER 67 | 68 | #endif 69 | -------------------------------------------------------------------------------- /inc/ix/stddef.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * stddef.h - standard definitions 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | #include 32 | #include 33 | 34 | /* 35 | * NOTE: Some code in this file is derived from the public domain CCAN project. 36 | * http://ccodearchive.net/ 37 | */ 38 | 39 | /** 40 | * BUILD_ASSERT - assert a build-time dependency. 41 | * @cond: the compile-time condition which must be true. 42 | * 43 | * Your compile will fail if the condition isn't true, or can't be evaluated 44 | * by the compiler. This can only be used within a function. 45 | */ 46 | #define BUILD_ASSERT(cond) \ 47 | do { (void) sizeof(char [1 - 2*!(cond)]); } while(0) 48 | 49 | /** 50 | * BUILD_ASSERT_OR_ZERO - assert a build-time dependency, as an expression. 51 | * @cond: the compile-time condition which must be true. 52 | * 53 | * Your compile will fail if the condition isn't true, or can't be evaluated 54 | * by the compiler. This can be used in an expression: its value is "0". 55 | */ 56 | #define BUILD_ASSERT_OR_ZERO(cond) \ 57 | (sizeof(char [1 - 2*!(cond)]) - 1) 58 | 59 | #define check_type(expr, type) \ 60 | ((typeof(expr) *)0 != (type *)0) 61 | #define check_types_match(expr1, expr2) \ 62 | ((typeof(expr1) *)0 != (typeof(expr2) *)0) 63 | 64 | /** 65 | * container_of - get pointer to enclosing structure 66 | * @member_ptr: pointer to the structure member 67 | * @containing_type: the type this member is within 68 | * @member: the name of this member within the structure. 69 | * 70 | * Given a pointer to a member of a structure, this macro does pointer 71 | * subtraction to return the pointer to the enclosing type. 72 | */ 73 | #define container_of(member_ptr, containing_type, member) \ 74 | ((containing_type *) \ 75 | ((char *)(member_ptr) \ 76 | - offsetof(containing_type, member)) \ 77 | + check_types_match(*(member_ptr), ((containing_type *)0)->member)) 78 | 79 | /** 80 | * container_of_var - get pointer to enclosing structure using a variable 81 | * @member_ptr: pointer to the structure member 82 | * @container_var: a pointer of same type as this member's container 83 | * @member: the name of this member within the structure. 84 | */ 85 | #define container_of_var(member_ptr, container_var, member) \ 86 | container_of(member_ptr, typeof(*container_var), member) 87 | 88 | #define _array_size_chk(arr) \ 89 | BUILD_ASSERT_OR_ZERO(!__builtin_types_compatible_p(typeof(arr), \ 90 | typeof(&(arr)[0]))) 91 | 92 | /** 93 | * ARRAY_SIZE - get the number of elements in a visible array 94 | * @arr: the array whose size you want. 95 | * 96 | * This does not work on pointers, or arrays declared as [], or 97 | * function parameters. With correct compiler support, such usage 98 | * will cause a build error (see build_assert). 99 | */ 100 | #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]) + _array_size_chk(arr)) 101 | 102 | /** 103 | * max - picks the maximum of two expressions 104 | * 105 | * Arguments @a and @b are evaluated exactly once 106 | */ 107 | #define max(a, b) \ 108 | ({typeof(a) _a = (a); \ 109 | typeof(b) _b = (b); \ 110 | _a > _b ? _a : _b; }) 111 | 112 | /** 113 | * min - picks the minimum of two expressions 114 | * 115 | * Arguments @a and @b are evaluated exactly once 116 | */ 117 | #define min(a, b) \ 118 | ({typeof(a) _a = (a); \ 119 | typeof(b) _b = (b); \ 120 | _a < _b ? _a : _b; }) 121 | 122 | /** 123 | * align_up - rounds a value up to an alignment 124 | * @x: the value 125 | * @a: the alignment (must be power of 2) 126 | * 127 | * Returns an aligned value. 128 | */ 129 | #define align_up(x, a) ((((x) - 1) | ((typeof(x)) (a) - 1)) + 1) 130 | 131 | /** 132 | * align_down - rounds a value down to an alignment 133 | * @x: the value 134 | * @a: the alignment (must be power of 2) 135 | * 136 | * Returns an aligned value. 137 | */ 138 | #define align_down(x, a) ((x) & ~((typeof(x)) (a) - 1)) 139 | 140 | /** 141 | * div_up - divides two numbers, rounding up to an integer 142 | * @x: the dividend 143 | * @d: the divisor 144 | * 145 | * Returns a rounded-up quotient. 146 | */ 147 | #define div_up(x, d) ((((x) + (d) - 1)) / (d)) 148 | 149 | -------------------------------------------------------------------------------- /inc/ix/tailqueue.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | 25 | #pragma once 26 | 27 | 28 | #define MIN_NINES 1 /* 90% */ 29 | #define MAX_NINES 5 /* 99.999 */ 30 | 31 | const char *tailqueue_nines[] = {"", "90%", "99%", "99.9%", "99.99%", "99.999"}; 32 | 33 | 34 | struct tailqueue; 35 | 36 | struct taildistr { 37 | uint64_t count, min, max; 38 | uint64_t nines[MAX_NINES + 1]; /* nb: nines[0] is unused */ 39 | }; 40 | 41 | extern void tailqueue_addsample(struct tailqueue *tq, 42 | uint64_t t_us); 43 | 44 | extern void tailqueue_calcnines(struct tailqueue *tq, 45 | struct taildistr *td, 46 | int reset); 47 | 48 | 49 | -------------------------------------------------------------------------------- /inc/ix/timer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * timer.h - timer event infrastructure 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | 32 | struct eth_fg; 33 | 34 | struct timer { 35 | struct hlist_node link; 36 | void (*handler)(struct timer *t, struct eth_fg *cur_fg); 37 | uint64_t expires; 38 | int fg_id; 39 | }; 40 | 41 | 42 | #define ONE_SECOND 1000000 43 | #define ONE_MS 1000 44 | #define ONE_US 1 45 | /** 46 | * timer_init_entry - initializes a timer 47 | * @t: the timer 48 | */ 49 | static inline void 50 | timer_init_entry(struct timer *t, void (*handler)(struct timer *t, struct eth_fg *)) 51 | { 52 | t->link.prev = NULL; 53 | t->handler = handler; 54 | } 55 | 56 | /** 57 | * timer_pending - determines if a timer is pending 58 | * @t: the timer 59 | * 60 | * Returns true if the timer is pending, otherwise false. 61 | */ 62 | static inline bool timer_pending(struct timer *t) 63 | { 64 | return t->link.prev != NULL; 65 | } 66 | 67 | extern int timer_add(struct timer *t, struct eth_fg *, uint64_t usecs); 68 | extern void timer_add_for_next_tick(struct timer *t, struct eth_fg *); 69 | extern void timer_add_abs(struct timer *t, struct eth_fg *, uint64_t usecs); 70 | extern uint64_t timer_now(void); 71 | 72 | static inline void __timer_del(struct timer *t) 73 | { 74 | hlist_del(&t->link); 75 | t->link.prev = NULL; 76 | } 77 | 78 | /** 79 | * timer_mod - modifies a timer 80 | * @t: the timer 81 | * @usecs: the number of microseconds from present to fire the timer 82 | * 83 | * If the timer is already armed, then its trigger time is modified. 84 | * Otherwise this function behaves like timer_add(). 85 | * 86 | * Returns 0 if successful, otherwise failure. 87 | */ 88 | static inline int timer_mod(struct timer *t, struct eth_fg *cur_fg, uint64_t usecs) 89 | { 90 | if (timer_pending(t)) 91 | __timer_del(t); 92 | return timer_add(t, cur_fg, usecs); 93 | } 94 | 95 | /** 96 | * timer_del - disarms a timer 97 | * @t: the timer 98 | * 99 | * If the timer is already disarmed, then nothing happens. 100 | */ 101 | static inline void timer_del(struct timer *t) 102 | { 103 | if (timer_pending(t)) 104 | __timer_del(t); 105 | } 106 | 107 | extern void timer_run(void); 108 | extern uint64_t timer_deadline(uint64_t max_us); 109 | 110 | extern int timer_collect_fgs(uint8_t *fg_vector, struct hlist_head *list, uint64_t *timer_pos); 111 | extern void timer_reinject_fgs(struct hlist_head *list, uint64_t timer_pos); 112 | 113 | 114 | extern void timer_init_fg(void); 115 | extern int timer_init_cpu(void); 116 | extern int timer_init(void); 117 | 118 | extern int cycles_per_us; 119 | 120 | 121 | 122 | -------------------------------------------------------------------------------- /inc/ix/types.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * types.h - primitive type definitions 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | #include 32 | 33 | typedef unsigned char uint8_t; 34 | typedef unsigned short uint16_t; 35 | typedef unsigned int uint32_t; 36 | 37 | typedef signed char int8_t; 38 | typedef signed short int16_t; 39 | typedef signed int int32_t; 40 | 41 | #ifndef __WORD_SIZE 42 | #error __WORD_SIZE is undefined 43 | #endif 44 | 45 | #if __WORD_SIZE == __64BIT_WORDS 46 | 47 | typedef unsigned long uint64_t; 48 | typedef signed long int64_t; 49 | 50 | #else /* __WORDSIZE == __64BIT_WORDS */ 51 | 52 | typedef unsigned long long uint64_t; 53 | typedef signed long long int64_t; 54 | 55 | #endif /* __WORDSIZE == __64BIT_WORDS */ 56 | 57 | typedef unsigned long uintptr_t; 58 | typedef long off_t; 59 | typedef unsigned long size_t; 60 | typedef long ssize_t; 61 | 62 | typedef struct { 63 | volatile int locked; 64 | } spinlock_t; 65 | 66 | typedef struct { 67 | int cnt; 68 | } atomic_t; 69 | 70 | typedef struct { 71 | long cnt; 72 | } atomic64_t; 73 | 74 | -------------------------------------------------------------------------------- /inc/ix/uaccess.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * uaccess.h - routines for safely accessing user memory 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | #include 32 | #include 33 | 34 | #include 35 | 36 | /** 37 | * uaccess_okay - determines if a memory object lies in userspace 38 | * @addr: the address 39 | * @len: the length of the memory object 40 | * 41 | * Returns true if the memory is user-level, otherwise false. 42 | */ 43 | static inline bool uaccess_okay(void *addr, size_t len) 44 | { 45 | if (len > MEM_USER_END - MEM_USER_START || 46 | (uintptr_t) addr < MEM_USER_START || 47 | (uintptr_t) addr + len > MEM_USER_END) 48 | return false; 49 | return true; 50 | } 51 | 52 | /** 53 | * uaccess_zc_okay - determines if a memory object is safe to zero-copy 54 | * @addr: the address 55 | * @len: the length of the memory object 56 | * 57 | * Returns true if the memory is safe for zero-copy, otherwise false. 58 | */ 59 | static inline bool uaccess_zc_okay(void *addr, size_t len) 60 | { 61 | if (len > MEM_ZC_USER_END - MEM_ZC_USER_START || 62 | (uintptr_t) addr < MEM_ZC_USER_START || 63 | (uintptr_t) addr + len > MEM_ZC_USER_END) 64 | return false; 65 | return true; 66 | } 67 | 68 | /** 69 | * copy_from_user - safely copies user memory to kernel memory 70 | * @user_src: the user source memory 71 | * @kern_dst: the kernel destination memory 72 | * @len: the number of bytes to copy 73 | * 74 | * Returns 0 if successful, otherwise -EFAULT if unsafe. 75 | */ 76 | static inline int copy_from_user(void *user_src, void *kern_dst, size_t len) 77 | { 78 | if (!uaccess_okay(user_src, len)) 79 | return -EFAULT; 80 | 81 | if (__builtin_constant_p(len)) { 82 | if (len == sizeof(uint64_t)) { 83 | *((uint64_t *) kern_dst) = uaccess_peekq(user_src); 84 | return uaccess_check_fault() ? -EFAULT : 0; 85 | } 86 | } 87 | 88 | return uaccess_copy_user(user_src, kern_dst, len); 89 | } 90 | 91 | /** 92 | * copy_to_user - safely copies kernel memory to user memory 93 | * @kern_src: the kernel source memory 94 | * @user_dst: the user destination memory 95 | * @len: the number of bytes to copy 96 | * 97 | * Returns 0 if successful, otherwise -EFAULT if unsafe. 98 | */ 99 | static inline int copy_to_user(void *kern_src, void *user_dst, size_t len) 100 | { 101 | if (!uaccess_okay(user_dst, len)) 102 | return -EFAULT; 103 | 104 | if (__builtin_constant_p(len)) { 105 | if (len == sizeof(uint64_t)) { 106 | uaccess_pokeq(user_dst, *((uint64_t *) kern_src)); 107 | return uaccess_check_fault() ? -EFAULT : 0; 108 | } 109 | } 110 | 111 | return uaccess_copy_user(kern_src, user_dst, len); 112 | } 113 | 114 | -------------------------------------------------------------------------------- /inc/ix/utimer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | struct utimer_list; 6 | 7 | DECLARE_PERCPU(struct utimer_list, utimers); 8 | 9 | int utimer_init(struct utimer_list *tl, void *udata); 10 | 11 | int utimer_arm(struct utimer_list *tl, int timer_id, uint64_t delay); 12 | -------------------------------------------------------------------------------- /inc/ix/vm.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * vm.h - virtual memory management support 26 | * 27 | * NOTE: right now we mostly use Dune for these operations, so this file 28 | * serves only to augment those capabilities. 29 | */ 30 | 31 | #pragma once 32 | 33 | #define VM_PERM_R 0x1 34 | #define VM_PERM_W 0x2 35 | #define VM_PERM_X 0x4 36 | #define VM_PERM_U 0x8 37 | 38 | #ifdef __KERNEL__ 39 | 40 | #include 41 | #include 42 | 43 | /* FIXME: this should be defined in inc/asm */ 44 | #include 45 | 46 | DECLARE_SPINLOCK(vm_lock); 47 | 48 | /* FIXME: a bunch of gross hacks until we can better integrate libdune */ 49 | #define UINT64(x) ((uint64_t) x) 50 | #define CAST64(x) ((uint64_t) x) 51 | typedef uint64_t ptent_t; 52 | #define NPTBITS 9 53 | #ifndef pgroot 54 | extern ptent_t *pgroot; 55 | #endif 56 | 57 | /** 58 | * vm_lookup_phys - determine a physical address from a virtual address 59 | * @virt: the virtual address 60 | * @pgsize: the size of the page at the address (must be correct). 61 | * 62 | * Returns a physical address. 63 | */ 64 | static inline physaddr_t vm_lookup_phys(void *virt, int pgsize) 65 | { 66 | ptent_t *dir = pgroot; 67 | ptent_t pte; 68 | 69 | pte = dir[PDX(3, virt)]; 70 | if (!(PTE_FLAGS(pte) & PTE_P)) 71 | return 0; 72 | 73 | dir = (ptent_t *) PTE_ADDR(pte); 74 | pte = dir[PDX(2, virt)]; 75 | if (!(PTE_FLAGS(pte) & PTE_P)) 76 | return 0; 77 | if (pgsize == PGSIZE_1GB) 78 | return (physaddr_t) PTE_ADDR(pte); 79 | 80 | dir = (ptent_t *) PTE_ADDR(pte); 81 | pte = dir[PDX(1, virt)]; 82 | if (!(PTE_FLAGS(pte) & PTE_P)) 83 | return 0; 84 | if (pgsize == PGSIZE_2MB) 85 | return (physaddr_t) PTE_ADDR(pte); 86 | 87 | dir = (ptent_t *) PTE_ADDR(pte); 88 | pte = dir[PDX(0, virt)]; 89 | if (!(PTE_FLAGS(pte) & PTE_P)) 90 | return 0; 91 | return (physaddr_t) PTE_ADDR(pte); 92 | } 93 | 94 | extern int __vm_map_phys(physaddr_t pa, virtaddr_t va, 95 | int nr, int size, int perm); 96 | extern bool __vm_is_mapped(void *addr, size_t len); 97 | 98 | extern int vm_map_phys(physaddr_t pa, virtaddr_t va, 99 | int nr, int size, int perm); 100 | extern void *vm_map_to_user(void *kern_addr, int nr, int size, int perm); 101 | extern void vm_unmap(void *addr, int nr, int size); 102 | 103 | #endif /* __KERNEL__ */ 104 | 105 | -------------------------------------------------------------------------------- /inc/lwip/arch/cc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * Copyright (c) 2001-2003 Swedish Institute of Computer Science. 26 | * All rights reserved. 27 | * 28 | * Redistribution and use in source and binary forms, with or without modification, 29 | * are permitted provided that the following conditions are met: 30 | * 31 | * 1. Redistributions of source code must retain the above copyright notice, 32 | * this list of conditions and the following disclaimer. 33 | * 2. Redistributions in binary form must reproduce the above copyright notice, 34 | * this list of conditions and the following disclaimer in the documentation 35 | * and/or other materials provided with the distribution. 36 | * 3. The name of the author may not be used to endorse or promote products 37 | * derived from this software without specific prior written permission. 38 | * 39 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 40 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 41 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 42 | * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 43 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 44 | * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 45 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 46 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 47 | * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 48 | * OF SUCH DAMAGE. 49 | * 50 | * This file is part of the lwIP TCP/IP stack. 51 | * 52 | * Author: Adam Dunkels 53 | * 54 | */ 55 | #ifndef __ARCH_CC_H__ 56 | #define __ARCH_CC_H__ 57 | 58 | /* Include some files for defining library routines */ 59 | #include 60 | #include 61 | #include 62 | 63 | #define LWIP_TIMEVAL_PRIVATE 0 64 | 65 | /* Define platform endianness */ 66 | #ifndef BYTE_ORDER 67 | #define BYTE_ORDER LITTLE_ENDIAN 68 | #endif /* BYTE_ORDER */ 69 | 70 | /* Define generic types used in lwIP */ 71 | typedef unsigned char u8_t; 72 | typedef signed char s8_t; 73 | typedef unsigned short u16_t; 74 | typedef signed short s16_t; 75 | typedef unsigned int u32_t; 76 | typedef signed int s32_t; 77 | 78 | typedef unsigned long mem_ptr_t; 79 | 80 | /* Define (sn)printf formatters for these lwIP types */ 81 | #define X8_F "02x" 82 | #define U16_F "hu" 83 | #define S16_F "hd" 84 | #define X16_F "hx" 85 | #define U32_F "u" 86 | #define S32_F "d" 87 | #define X32_F "x" 88 | 89 | /* If only we could use C99 and get %zu */ 90 | #if defined(__x86_64__) 91 | #define SZT_F "lu" 92 | #else 93 | #define SZT_F "u" 94 | #endif 95 | 96 | /* Compiler hints for packing structures */ 97 | #define PACK_STRUCT_FIELD(x) x 98 | #define PACK_STRUCT_STRUCT __attribute__((packed)) 99 | #define PACK_STRUCT_BEGIN 100 | #define PACK_STRUCT_END 101 | 102 | /* prototypes for printf() and abort() */ 103 | #include 104 | #include 105 | /* Plaform specific diagnostic output */ 106 | #define LWIP_PLATFORM_DIAG(x) do {printf x;} while(0) 107 | 108 | #ifdef LWIP_UNIX_EMPTY_ASSERT 109 | #define LWIP_PLATFORM_ASSERT(x) 110 | #else 111 | #define LWIP_PLATFORM_ASSERT(x) do {printf("Assertion \"%s\" failed at line %d in %s\n", \ 112 | x, __LINE__, __FILE__); fflush(NULL); abort();} while(0) 113 | #endif 114 | 115 | #define LWIP_RAND() ((u32_t)rand()) 116 | 117 | #endif /* __ARCH_CC_H__ */ 118 | -------------------------------------------------------------------------------- /inc/lwip/arch/perf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * Copyright (c) 2001-2003 Swedish Institute of Computer Science. 26 | * All rights reserved. 27 | * 28 | * Redistribution and use in source and binary forms, with or without modification, 29 | * are permitted provided that the following conditions are met: 30 | * 31 | * 1. Redistributions of source code must retain the above copyright notice, 32 | * this list of conditions and the following disclaimer. 33 | * 2. Redistributions in binary form must reproduce the above copyright notice, 34 | * this list of conditions and the following disclaimer in the documentation 35 | * and/or other materials provided with the distribution. 36 | * 3. The name of the author may not be used to endorse or promote products 37 | * derived from this software without specific prior written permission. 38 | * 39 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 40 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 41 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 42 | * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 43 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 44 | * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 45 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 46 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 47 | * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 48 | * OF SUCH DAMAGE. 49 | * 50 | * This file is part of the lwIP TCP/IP stack. 51 | * 52 | * Author: Adam Dunkels 53 | * 54 | */ 55 | #ifndef __ARCH_PERF_H__ 56 | #define __ARCH_PERF_H__ 57 | 58 | #include 59 | 60 | #ifdef PERF 61 | #define PERF_START { \ 62 | unsigned long __c1l, __c1h, __c2l, __c2h; \ 63 | __asm__(".byte 0x0f, 0x31" : "=a" (__c1l), "=d" (__c1h)) 64 | #define PERF_STOP(x) __asm__(".byte 0x0f, 0x31" : "=a" (__c2l), "=d" (__c2h)); \ 65 | perf_print(__c1l, __c1h, __c2l, __c2h, x);} 66 | 67 | /*#define PERF_START do { \ 68 | struct tms __perf_start, __perf_end; \ 69 | times(&__perf_start) 70 | #define PERF_STOP(x) times(&__perf_end); \ 71 | perf_print_times(&__perf_start, &__perf_end, x);\ 72 | } while(0)*/ 73 | #else /* PERF */ 74 | #define PERF_START /* null definition */ 75 | #define PERF_STOP(x) /* null definition */ 76 | #endif /* PERF */ 77 | 78 | void perf_print(unsigned long c1l, unsigned long c1h, 79 | unsigned long c2l, unsigned long c2h, 80 | char *key); 81 | 82 | void perf_print_times(struct tms *start, struct tms *end, char *key); 83 | 84 | void perf_init(char *fname); 85 | 86 | #endif /* __ARCH_PERF_H__ */ 87 | -------------------------------------------------------------------------------- /inc/lwip/lwip/err.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * Copyright (c) 2001-2004 Swedish Institute of Computer Science. 26 | * All rights reserved. 27 | * 28 | * Redistribution and use in source and binary forms, with or without modification, 29 | * are permitted provided that the following conditions are met: 30 | * 31 | * 1. Redistributions of source code must retain the above copyright notice, 32 | * this list of conditions and the following disclaimer. 33 | * 2. Redistributions in binary form must reproduce the above copyright notice, 34 | * this list of conditions and the following disclaimer in the documentation 35 | * and/or other materials provided with the distribution. 36 | * 3. The name of the author may not be used to endorse or promote products 37 | * derived from this software without specific prior written permission. 38 | * 39 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 40 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 41 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 42 | * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 43 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 44 | * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 45 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 46 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 47 | * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 48 | * OF SUCH DAMAGE. 49 | * 50 | * This file is part of the lwIP TCP/IP stack. 51 | * 52 | * Author: Adam Dunkels 53 | * 54 | */ 55 | #ifndef __LWIP_ERR_H__ 56 | #define __LWIP_ERR_H__ 57 | 58 | #include "lwip/opt.h" 59 | #include "lwip/arch.h" 60 | 61 | #ifdef __cplusplus 62 | extern "C" { 63 | #endif 64 | 65 | /** Define LWIP_ERR_T in cc.h if you want to use 66 | * a different type for your platform (must be signed). */ 67 | #ifdef LWIP_ERR_T 68 | typedef LWIP_ERR_T err_t; 69 | #else /* LWIP_ERR_T */ 70 | typedef s8_t err_t; 71 | #endif /* LWIP_ERR_T*/ 72 | 73 | /* Definitions for error constants. */ 74 | 75 | #define ERR_OK 0 /* No error, everything OK. */ 76 | #define ERR_MEM -1 /* Out of memory error. */ 77 | #define ERR_BUF -2 /* Buffer error. */ 78 | #define ERR_TIMEOUT -3 /* Timeout. */ 79 | #define ERR_RTE -4 /* Routing problem. */ 80 | #define ERR_INPROGRESS -5 /* Operation in progress */ 81 | #define ERR_VAL -6 /* Illegal value. */ 82 | #define ERR_WOULDBLOCK -7 /* Operation would block. */ 83 | #define ERR_USE -8 /* Address in use. */ 84 | #define ERR_ISCONN -9 /* Already connected. */ 85 | 86 | #define ERR_IS_FATAL(e) ((e) < ERR_ISCONN) 87 | 88 | #define ERR_ABRT -10 /* Connection aborted. */ 89 | #define ERR_RST -11 /* Connection reset. */ 90 | #define ERR_CLSD -12 /* Connection closed. */ 91 | #define ERR_CONN -13 /* Not connected. */ 92 | 93 | #define ERR_ARG -14 /* Illegal argument. */ 94 | 95 | #define ERR_IF -15 /* Low-level netif error */ 96 | 97 | 98 | #ifdef LWIP_DEBUG 99 | extern const char *lwip_strerr(err_t err); 100 | #else 101 | #define lwip_strerr(x) "" 102 | #endif /* LWIP_DEBUG */ 103 | 104 | #ifdef __cplusplus 105 | } 106 | #endif 107 | 108 | #endif /* __LWIP_ERR_H__ */ 109 | -------------------------------------------------------------------------------- /inc/lwip/lwip/mem.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | #ifndef __LWIP_MEM_H__ 25 | #define __LWIP_MEM_H__ 26 | 27 | typedef size_t mem_size_t; 28 | 29 | /** mem_init is not used when using pools instead of a heap */ 30 | #define mem_init() 31 | /** mem_trim is not used when using pools instead of a heap: 32 | we can't free part of a pool element and don't want to copy the rest */ 33 | #define mem_trim(mem, size) (mem) 34 | void *mem_malloc(mem_size_t size); 35 | void mem_free(void *mem); 36 | 37 | /** Calculate memory size for an aligned buffer - returns the next highest 38 | * multiple of MEM_ALIGNMENT (e.g. LWIP_MEM_ALIGN_SIZE(3) and 39 | * LWIP_MEM_ALIGN_SIZE(4) will both yield 4 for MEM_ALIGNMENT == 4). 40 | */ 41 | #ifndef LWIP_MEM_ALIGN_SIZE 42 | #define LWIP_MEM_ALIGN_SIZE(size) (((size) + MEM_ALIGNMENT - 1) & ~(MEM_ALIGNMENT-1)) 43 | #endif 44 | 45 | /** Align a memory pointer to the alignment defined by MEM_ALIGNMENT 46 | * so that ADDR % MEM_ALIGNMENT == 0 47 | */ 48 | #ifndef LWIP_MEM_ALIGN 49 | #define LWIP_MEM_ALIGN(addr) ((void *)(((mem_ptr_t)(addr) + MEM_ALIGNMENT - 1) & ~(mem_ptr_t)(MEM_ALIGNMENT-1))) 50 | #endif 51 | 52 | #endif /* __LWIP_MEM_H__ */ 53 | -------------------------------------------------------------------------------- /inc/lwip/lwip/memp.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | #ifndef __LWIP_MEMP_H__ 25 | #define __LWIP_MEMP_H__ 26 | 27 | #include 28 | #include 29 | #include 30 | 31 | #include "lwip/opt.h" 32 | 33 | /* Create the list of all memory pools managed by memp. MEMP_MAX represents a NULL pool at the end */ 34 | typedef enum { 35 | #define LWIP_MEMPOOL(name,num,size,desc) MEMP_##name, 36 | #include "lwip/memp_std.h" 37 | MEMP_MAX 38 | } memp_t; 39 | 40 | extern const u16_t memp_sizes[MEMP_MAX]; 41 | 42 | int memp_init(void); 43 | int memp_init_cpu(void); 44 | 45 | DECLARE_PERCPU(struct mempool, pbuf_mempool); 46 | DECLARE_PERCPU(struct mempool, pbuf_with_payload_mempool); 47 | DECLARE_PERCPU(struct mempool, tcp_pcb_mempool); 48 | DECLARE_PERCPU(struct mempool, tcp_pcb_listen_mempool); 49 | DECLARE_PERCPU(struct mempool, tcp_seg_mempool); 50 | 51 | static inline void *memp_malloc(memp_t type) 52 | { 53 | switch (type) { 54 | case MEMP_PBUF: 55 | return mempool_alloc(&percpu_get(pbuf_mempool)); 56 | case MEMP_TCP_PCB: 57 | return mempool_alloc(&percpu_get(tcp_pcb_mempool)); 58 | case MEMP_TCP_PCB_LISTEN: 59 | return mempool_alloc(&percpu_get(tcp_pcb_listen_mempool)); 60 | case MEMP_TCP_SEG: 61 | return mempool_alloc(&percpu_get(tcp_seg_mempool)); 62 | case MEMP_SYS_TIMEOUT: 63 | case MEMP_PBUF_POOL: 64 | case MEMP_MAX: 65 | break; 66 | } 67 | 68 | return NULL; 69 | } 70 | 71 | static inline void memp_free(memp_t type, void *mem) 72 | { 73 | switch (type) { 74 | case MEMP_PBUF: 75 | mempool_free(&percpu_get(pbuf_mempool), mem); 76 | return; 77 | case MEMP_TCP_PCB: 78 | mempool_free(&percpu_get(tcp_pcb_mempool), mem); 79 | return; 80 | case MEMP_TCP_PCB_LISTEN: 81 | mempool_free(&percpu_get(tcp_pcb_listen_mempool), mem); 82 | return; 83 | case MEMP_TCP_SEG: 84 | mempool_free(&percpu_get(tcp_seg_mempool), mem); 85 | return; 86 | case MEMP_SYS_TIMEOUT: 87 | case MEMP_PBUF_POOL: 88 | case MEMP_MAX: 89 | break; 90 | } 91 | } 92 | 93 | #endif /* __LWIP_MEMP_H__ */ 94 | -------------------------------------------------------------------------------- /inc/lwip/lwip/timers.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * Copyright (c) 2001-2004 Swedish Institute of Computer Science. 26 | * All rights reserved. 27 | * 28 | * Redistribution and use in source and binary forms, with or without modification, 29 | * are permitted provided that the following conditions are met: 30 | * 31 | * 1. Redistributions of source code must retain the above copyright notice, 32 | * this list of conditions and the following disclaimer. 33 | * 2. Redistributions in binary form must reproduce the above copyright notice, 34 | * this list of conditions and the following disclaimer in the documentation 35 | * and/or other materials provided with the distribution. 36 | * 3. The name of the author may not be used to endorse or promote products 37 | * derived from this software without specific prior written permission. 38 | * 39 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 40 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 41 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 42 | * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 43 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 44 | * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 45 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 46 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 47 | * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 48 | * OF SUCH DAMAGE. 49 | * 50 | * This file is part of the lwIP TCP/IP stack. 51 | * 52 | * Author: Adam Dunkels 53 | * Simon Goldschmidt 54 | * 55 | */ 56 | #ifndef __LWIP_TIMERS_H__ 57 | #define __LWIP_TIMERS_H__ 58 | 59 | #include "lwip/opt.h" 60 | 61 | /* Timers are not supported when NO_SYS==1 and NO_SYS_NO_TIMERS==1 */ 62 | #define LWIP_TIMERS (!NO_SYS || (NO_SYS && !NO_SYS_NO_TIMERS)) 63 | 64 | #if LWIP_TIMERS 65 | 66 | #include "lwip/err.h" 67 | #if !NO_SYS 68 | #include "lwip/sys.h" 69 | #endif 70 | 71 | #ifdef __cplusplus 72 | extern "C" { 73 | #endif 74 | 75 | #ifndef LWIP_DEBUG_TIMERNAMES 76 | #ifdef LWIP_DEBUG 77 | #define LWIP_DEBUG_TIMERNAMES SYS_DEBUG 78 | #else /* LWIP_DEBUG */ 79 | #define LWIP_DEBUG_TIMERNAMES 0 80 | #endif /* LWIP_DEBUG*/ 81 | #endif 82 | 83 | /** Function prototype for a timeout callback function. Register such a function 84 | * using sys_timeout(). 85 | * 86 | * @param arg Additional argument to pass to the function - set up by sys_timeout() 87 | */ 88 | typedef void (* sys_timeout_handler)(void *arg); 89 | 90 | struct sys_timeo { 91 | struct sys_timeo *next; 92 | u32_t time; 93 | sys_timeout_handler h; 94 | void *arg; 95 | #if LWIP_DEBUG_TIMERNAMES 96 | const char* handler_name; 97 | #endif /* LWIP_DEBUG_TIMERNAMES */ 98 | }; 99 | 100 | void sys_timeouts_init(void); 101 | 102 | #if LWIP_DEBUG_TIMERNAMES 103 | void sys_timeout_debug(u32_t msecs, sys_timeout_handler handler, void *arg, const char* handler_name); 104 | #define sys_timeout(msecs, handler, arg) sys_timeout_debug(msecs, handler, arg, #handler) 105 | #else /* LWIP_DEBUG_TIMERNAMES */ 106 | void sys_timeout(u32_t msecs, sys_timeout_handler handler, void *arg); 107 | #endif /* LWIP_DEBUG_TIMERNAMES */ 108 | 109 | void sys_untimeout(sys_timeout_handler handler, void *arg); 110 | #if NO_SYS 111 | void sys_check_timeouts(void); 112 | void sys_restart_timeouts(void); 113 | #else /* NO_SYS */ 114 | void sys_timeouts_mbox_fetch(sys_mbox_t *mbox, void **msg); 115 | #endif /* NO_SYS */ 116 | 117 | 118 | #ifdef __cplusplus 119 | } 120 | #endif 121 | 122 | #endif /* LWIP_TIMERS */ 123 | #endif /* __LWIP_TIMERS_H__ */ 124 | -------------------------------------------------------------------------------- /inc/lwip/lwipopts.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | #define LWIP_STATS 0 25 | #define LWIP_TCP 1 26 | #define NO_SYS 1 27 | #define LWIP_RAW 0 28 | #define LWIP_UDP 0 29 | #define IP_REASSEMBLY 0 30 | #define IP_FRAG 0 31 | #define LWIP_NETCONN 0 32 | 33 | #define MEM_LIBC_MALLOC 1 34 | #define MEMP_MEM_MALLOC 1 35 | 36 | //#define LWIP_DEBUG LWIP_DBG_OFF 37 | #undef LWIP_DEBUG 38 | #define TCP_CWND_DEBUG LWIP_DBG_OFF 39 | #define TCP_DEBUG LWIP_DBG_OFF 40 | #define TCP_FR_DEBUG LWIP_DBG_OFF 41 | #define TCP_INPUT_DEBUG LWIP_DBG_OFF 42 | #define TCP_OUTPUT_DEBUG LWIP_DBG_OFF 43 | #define TCP_QLEN_DEBUG LWIP_DBG_OFF 44 | #define TCP_RST_DEBUG LWIP_DBG_OFF 45 | #define TCP_RTO_DEBUG LWIP_DBG_OFF 46 | #define TCP_WND_DEBUG LWIP_DBG_OFF 47 | 48 | #include 49 | #include 50 | 51 | #define LWIP_IX 52 | 53 | #define LWIP_PLATFORM_BYTESWAP 1 54 | #define LWIP_PLATFORM_HTONS(x) hton16(x) 55 | #define LWIP_PLATFORM_NTOHS(x) ntoh16(x) 56 | #define LWIP_PLATFORM_HTONL(x) hton32(x) 57 | #define LWIP_PLATFORM_NTOHL(x) ntoh32(x) 58 | 59 | #define LWIP_WND_SCALE 1 60 | #define TCP_RCV_SCALE 7 61 | #define TCP_SND_BUF 65536 62 | #define TCP_MSS 1460 63 | #define TCP_WND (2048 * TCP_MSS) 64 | 65 | #define CHECKSUM_CHECK_IP 0 66 | #define CHECKSUM_CHECK_TCP 0 67 | #define TCP_ACK_DELAY (1 * ONE_MS) 68 | #define RTO_UNITS (500 * ONE_MS) 69 | 70 | /* EdB 2014-11-07 */ 71 | #define LWIP_NOASSERT 72 | #define LWIP_EVENT_API 1 73 | #define LWIP_NETIF_HWADDRHINT 1 74 | 75 | -------------------------------------------------------------------------------- /inc/net/arp.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * arp.h - Address Resolution Protocol (RFC 826, RFC 903) 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | #include 32 | 33 | struct arp_hdr { 34 | uint16_t htype; 35 | uint16_t ptype; /* the ETHERTYPE */ 36 | uint8_t hlen; 37 | uint8_t plen; 38 | uint16_t op; 39 | 40 | /* 41 | * Variable length fields continue as follows: 42 | * sender hw addr: hlen bytes 43 | * sender protocol addr: plen bytes 44 | * target hw addr: hlen bytes 45 | * target protocol addr: plen bytes 46 | */ 47 | } __packed; 48 | 49 | struct arp_hdr_ethip { 50 | struct eth_addr sender_mac; 51 | struct ip_addr sender_ip; 52 | struct eth_addr target_mac; 53 | struct ip_addr target_ip; 54 | } __packed; 55 | 56 | #define ARP_HTYPE_ETHER 1 /* ethernet */ 57 | #define ARP_HTYPE_IEEE802 6 /* token-ring */ 58 | #define ARP_HTYPE_ARCNET 7 /* arcnet */ 59 | #define ARP_HTYPE_FRELAY 16 /* frame relay */ 60 | #define ARP_HTYPE_IEEE1394 24 /* firewire */ 61 | #define ARP_HTYPE_INFINIBAND 32 /* infiniband */ 62 | 63 | enum { 64 | ARP_OP_REQUEST = 1, /* request hw addr given protocol addr */ 65 | ARP_OP_REPLY = 2, /* response hw addr given protocol addr */ 66 | ARP_OP_REVREQUEST = 3, /* request protocol addr given hw addr */ 67 | ARP_OP_REVREPLY = 4, /* response protocol addr given hw addr */ 68 | }; 69 | 70 | -------------------------------------------------------------------------------- /inc/net/udp.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * udp.h - Unreliable Datagram Protocol 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | 32 | struct udp_hdr { 33 | uint16_t src_port; 34 | uint16_t dst_port; 35 | uint16_t len; 36 | uint16_t chksum; 37 | }; 38 | 39 | -------------------------------------------------------------------------------- /ix.conf.sample: -------------------------------------------------------------------------------- 1 | # ix.conf 2 | # configuration file for IX 3 | # 4 | # This file defines the configuration for IX data center OS. You should 5 | # carefully update all fields in accordence with your local configuration 6 | # before running IX for the first time. 7 | 8 | ############################################################################### 9 | # Network Parameters 10 | ############################################################################### 11 | 12 | ## host_addr : CIDR IP address and netmask that will be assigned to 13 | ## the adapter once it comes up. 14 | host_addr="192.168.40.1/16" 15 | 16 | # gateway_addr : default gateway IP address 17 | gateway_addr="192.168.1.1" 18 | 19 | ## port : port(s) that will be bound to the application by the 20 | ## kernel when launching IX. 21 | ## You can specify multiple entries, e.g. 'port=[X, Y, Z]' 22 | port=1234 23 | 24 | ############################################################################### 25 | # Hardware parameters 26 | ############################################################################### 27 | 28 | ## devices : Specifies the PCI device ID of the adapter to bind IX with. 29 | ## Should be an Intel compatible NIC (e.g. 82599, X520, X540, etc).. 30 | ## See the Docs to see which adapters are currently supported. 31 | ## Format is a list dddd:bb:ss.ff,... d - domain, b = bus, 32 | ## s = slot, f = function. Usually, `lspci | grep Ethernet` allows to see 33 | ## available Ethernet controllers. 34 | ## You can specify multiple entries, e.g. 'devices=["X","Y","Z"]' 35 | devices="0:05:00.0" 36 | 37 | ## cpu : Indicates which CPU process unit(s) (P) this IX instance 38 | ## should be bound to. 39 | ## WARNING: If this setting defines multiple nodes, make sure they ar 40 | ## part of the same NUMA socket. On most architectures, the IDs of 41 | ## processing units on the same socket are interleaved. 42 | ## Usually `lstopo` allows to see NUMA nodes topology 43 | ## You can specify multiple entries, e.g. 'nodes=["X","Y","Z"]' 44 | cpu=0 45 | 46 | ## batch : Specifies maximum batch size of received packets to process. 47 | ## Default: 64. 48 | batch=64 49 | 50 | ## loader_path : kernel loader to use with IX module: 51 | ## 52 | loader_path="/lib64/ld-linux-x86-64.so.2" 53 | 54 | 55 | ############################################################################### 56 | # Optional parameters 57 | ############################################################################### 58 | 59 | ## arp: allows you to manually add static arp entries in the interface arp 60 | ## table. 61 | #arp=( 62 | # { 63 | # ip : "192.168.1.2" 64 | # mac : "aa:aa:aa:aa:aa:aa" 65 | # }, 66 | # { 67 | # ip : "192.168.1.3" 68 | # mac : "bb:bb:bb:bb:bb:bb" 69 | # } 70 | #) 71 | 72 | 73 | -------------------------------------------------------------------------------- /libix/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2013-16 Board of Trustees of Stanford University 2 | # Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | # A Makefile for libIX. 23 | 24 | INC = -I. -I../inc 25 | CC = gcc 26 | CFLAGS = -g -Wall -O3 $(INC) 27 | AR = ar 28 | 29 | SRCS = main.c mem.c mempool.c ixev.c ixev_timer.c 30 | OBJS = $(subst .c,.o,$(SRCS)) 31 | 32 | all: libix.a 33 | 34 | depend: .depend 35 | 36 | .depend: $(SRCS) 37 | rm -f ./.depend 38 | $(foreach SRC,$(SRCS),$(CC) $(CFLAGS) -MM -MT $(SRC:.c=.o) $(SRC) >> .depend;) 39 | 40 | -include .depend 41 | 42 | libix.a: $(OBJS) 43 | $(AR) crD $(@) $(OBJS) 44 | 45 | clean: 46 | rm -f $(OBJS) libix.a .depend 47 | 48 | dist-clean: clean 49 | rm *~ 50 | 51 | -------------------------------------------------------------------------------- /libix/buf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * buf.h = transmit data buffer management 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | #include 32 | 33 | #include 34 | #include 35 | 36 | 37 | #define BUF_SIZE 1460 * 4 38 | 39 | extern __thread struct mempool ixev_buf_pool; 40 | 41 | struct ixev_buf { 42 | uint32_t len; 43 | uint32_t pad; 44 | struct ixev_ref ref; 45 | char payload[BUF_SIZE]; 46 | }; 47 | 48 | static inline void ixev_buf_release(struct ixev_ref *ref) 49 | { 50 | struct ixev_buf *buf = container_of(ref, struct ixev_buf, ref); 51 | mempool_free(&ixev_buf_pool, buf); 52 | } 53 | 54 | /** 55 | * ixev_buf_alloc - allocates a buffer 56 | * 57 | * The initial refcount is set to one. 58 | * 59 | * Returns a buffer, or NULL if out of memory. 60 | */ 61 | static inline struct ixev_buf *ixev_buf_alloc(void) 62 | { 63 | struct ixev_buf *buf = mempool_alloc(&ixev_buf_pool); 64 | 65 | if (unlikely(!buf)) 66 | return NULL; 67 | 68 | buf->len = 0; 69 | buf->ref.cb = &ixev_buf_release; 70 | 71 | return buf; 72 | } 73 | 74 | /** 75 | * ixev_buf_store - store data inside a buffer 76 | * @buf: the buffer 77 | * @addr: the start address of the data 78 | * @len: the length of the data 79 | * 80 | * Returns the numbers of bytes successfully stored in the buffer, 81 | * or zero if the buffer is full. 82 | */ 83 | static inline size_t ixev_buf_store(struct ixev_buf *buf, void *addr, size_t len) 84 | { 85 | size_t avail = min(len, BUF_SIZE - buf->len); 86 | 87 | if (!avail) 88 | return 0; 89 | 90 | memcpy(&buf->payload[buf->len], addr, avail); 91 | buf->len += avail; 92 | 93 | return avail; 94 | } 95 | 96 | /** 97 | * ixev_is_buf_full - determines if the buffer is full 98 | * @buf: the buffer 99 | * 100 | * Returns true if the buffer is full, otherwise false. 101 | */ 102 | static inline bool ixev_is_buf_full(struct ixev_buf *buf) 103 | { 104 | return buf->len == BUF_SIZE; 105 | } 106 | 107 | 108 | -------------------------------------------------------------------------------- /libix/ix.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * ix.h - the main libix header 26 | */ 27 | 28 | #pragma once 29 | 30 | #include "syscall.h" 31 | 32 | struct ix_ops { 33 | void (*udp_recv)(void *addr, size_t len, struct ip_tuple *id); 34 | void (*udp_sent)(unsigned long cookie); 35 | void (*tcp_connected)(hid_t handle, unsigned long cookie, 36 | long ret); 37 | void (*tcp_knock)(hid_t handle, struct ip_tuple *id); 38 | void (*tcp_recv)(hid_t handle, unsigned long cookie, 39 | void *addr, size_t len); 40 | void (*tcp_sent)(hid_t handle, unsigned long cookie, 41 | size_t win_size); 42 | void (*tcp_dead)(hid_t handle, unsigned long cookie); 43 | void (*timer_event)(unsigned long cookie); 44 | }; 45 | 46 | extern void ix_flush(void); 47 | extern __thread struct bsys_arr *karr; 48 | 49 | static inline int ix_bsys_idx(void) 50 | { 51 | return karr->len; 52 | } 53 | 54 | static inline void ix_udp_send(void *addr, size_t len, struct ip_tuple *id, 55 | unsigned long cookie) 56 | { 57 | if (karr->len >= karr->max_len) 58 | ix_flush(); 59 | 60 | ksys_udp_send(__bsys_arr_next(karr), addr, len, id, cookie); 61 | } 62 | 63 | static inline void ix_udp_sendv(struct sg_entry *ents, unsigned int nrents, 64 | struct ip_tuple *id, unsigned long cookie) 65 | { 66 | if (karr->len >= karr->max_len) 67 | ix_flush(); 68 | 69 | ksys_udp_sendv(__bsys_arr_next(karr), ents, nrents, id, cookie); 70 | } 71 | 72 | static inline void ix_udp_recv_done(void *addr) 73 | { 74 | if (karr->len >= karr->max_len) 75 | ix_flush(); 76 | 77 | ksys_udp_recv_done(__bsys_arr_next(karr), addr); 78 | } 79 | 80 | static inline void ix_tcp_connect(struct ip_tuple *id, unsigned long cookie) 81 | { 82 | if (karr->len >= karr->max_len) 83 | ix_flush(); 84 | 85 | ksys_tcp_connect(__bsys_arr_next(karr), id, cookie); 86 | } 87 | 88 | static inline void ix_tcp_accept(hid_t handle, unsigned long cookie) 89 | { 90 | if (karr->len >= karr->max_len) 91 | ix_flush(); 92 | 93 | ksys_tcp_accept(__bsys_arr_next(karr), handle, cookie); 94 | } 95 | 96 | static inline void ix_tcp_reject(hid_t handle) 97 | { 98 | if (karr->len >= karr->max_len) 99 | ix_flush(); 100 | 101 | ksys_tcp_reject(__bsys_arr_next(karr), handle); 102 | } 103 | 104 | static inline void ix_tcp_send(hid_t handle, void *addr, size_t len) 105 | { 106 | if (karr->len >= karr->max_len) 107 | ix_flush(); 108 | 109 | ksys_tcp_send(__bsys_arr_next(karr), handle, addr, len); 110 | } 111 | 112 | static inline void ix_tcp_sendv(hid_t handle, struct sg_entry *ents, 113 | unsigned int nrents) 114 | { 115 | if (karr->len >= karr->max_len) 116 | ix_flush(); 117 | 118 | ksys_tcp_sendv(__bsys_arr_next(karr), handle, ents, nrents); 119 | } 120 | 121 | static inline void ix_tcp_recv_done(hid_t handle, size_t len) 122 | { 123 | if (karr->len >= karr->max_len) 124 | ix_flush(); 125 | 126 | ksys_tcp_recv_done(__bsys_arr_next(karr), handle, len); 127 | } 128 | 129 | static inline void ix_tcp_close(hid_t handle) 130 | { 131 | if (karr->len >= karr->max_len) 132 | ix_flush(); 133 | 134 | ksys_tcp_close(__bsys_arr_next(karr), handle); 135 | } 136 | 137 | extern void *ix_alloc_pages(int nrpages); 138 | extern void ix_free_pages(void *addr, int nrpages); 139 | 140 | extern void ix_handle_events(void); 141 | extern int ix_poll(void); 142 | extern int ix_init(struct ix_ops *ops, int batch_depth); 143 | 144 | -------------------------------------------------------------------------------- /libix/ixev.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * ixev.h - a library that behaves mostly like libevent. 26 | */ 27 | 28 | #pragma once 29 | 30 | #include "ix.h" 31 | #include 32 | 33 | /* FIXME: we won't need recv depth when i get a chance to fix the kernel */ 34 | #define IXEV_RECV_DEPTH 128 35 | #define IXEV_SEND_DEPTH 16 36 | 37 | struct ixev_ctx; 38 | struct ixev_ref; 39 | struct ixev_buf; 40 | 41 | /* 42 | * FIXME: right now we only support some libevent features 43 | * - add level triggered support (now only edge triggered) 44 | * - add one fire events (now only persistent) 45 | * - add timeout support for events 46 | */ 47 | 48 | /* IX event types */ 49 | #define IXEVHUP 0x1 /* the connection was closed (or failed) */ 50 | #define IXEVIN 0x2 /* new data is available for reading */ 51 | #define IXEVOUT 0x4 /* more space is available for writing */ 52 | 53 | struct ixev_conn_ops { 54 | struct ixev_ctx *(*accept)(struct ip_tuple *id); 55 | void (*release)(struct ixev_ctx *ctx); 56 | void (*dialed)(struct ixev_ctx *ctx, long ret); 57 | }; 58 | 59 | /* 60 | * Use this callback to receive network event notifications 61 | */ 62 | typedef void (*ixev_handler_t)(struct ixev_ctx *ctx, unsigned int reason); 63 | 64 | /* 65 | * Use this callback to free memory after zero copy sends 66 | */ 67 | typedef void (*ixev_sent_cb_t)(struct ixev_ref *ref); 68 | 69 | struct ixev_ref { 70 | ixev_sent_cb_t cb; /* the decrement ref callback function */ 71 | size_t send_pos; /* the number of bytes sent before safe to free */ 72 | struct ixev_ref *next; /* the next ref in the sequence */ 73 | }; 74 | 75 | struct ixev_ctx { 76 | hid_t handle; /* the IX flow handle */ 77 | unsigned long user_data; /* application data */ 78 | uint64_t generation; /* generation number */ 79 | ixev_handler_t handler; /* the event handler */ 80 | unsigned int en_mask; /* a mask of enabled events */ 81 | unsigned int trig_mask; /* a mask of triggered events */ 82 | uint16_t recv_head; /* received data SG head */ 83 | uint16_t recv_tail; /* received data SG tail */ 84 | uint16_t send_count; /* the current send SG count */ 85 | uint16_t is_dead: 1; /* is the connection dead? */ 86 | 87 | size_t send_total; /* the total requested bytes */ 88 | size_t sent_total; /* the total completed bytes */ 89 | struct ixev_ref *ref_head; /* list head of references */ 90 | struct ixev_ref *ref_tail; /* list tail of references */ 91 | struct ixev_buf *cur_buf; /* current buffer */ 92 | 93 | struct bsys_desc *recv_done_desc; /* the current recv_done bsys descriptor */ 94 | struct bsys_desc *sendv_desc; /* the current sendv bsys descriptor */ 95 | 96 | struct sg_entry recv[IXEV_RECV_DEPTH]; /* receieve SG array */ 97 | struct sg_entry send[IXEV_SEND_DEPTH]; /* send SG array */ 98 | }; 99 | 100 | static inline void ixev_check_hacks(struct ixev_ctx *ctx) 101 | { 102 | /* 103 | * Temporary hack: 104 | * 105 | * FIXME: we need to flush commands in batches to limit our 106 | * command buffer size. Then this restriction can be lifted. 107 | */ 108 | if (unlikely(karr->len >= karr->max_len)) { 109 | printf("ixev: ran out of command space\n"); 110 | exit(-1); 111 | } 112 | } 113 | 114 | extern ssize_t ixev_recv(struct ixev_ctx *ctx, void *addr, size_t len); 115 | extern void *ixev_recv_zc(struct ixev_ctx *ctx, size_t len); 116 | extern ssize_t ixev_send(struct ixev_ctx *ctx, void *addr, size_t len); 117 | extern ssize_t ixev_send_zc(struct ixev_ctx *ctx, void *addr, size_t len); 118 | extern void ixev_add_sent_cb(struct ixev_ctx *ctx, struct ixev_ref *ref); 119 | 120 | extern void ixev_close(struct ixev_ctx *ctx); 121 | 122 | /** 123 | * ixev_dial - open a connection 124 | * @ctx: a freshly allocated and initialized context 125 | * @id: the address and port 126 | * @cb: the completion callback 127 | * 128 | * The completion returns a handle, or <0 if there was 129 | * an error. 130 | */ 131 | static inline void 132 | ixev_dial(struct ixev_ctx *ctx, struct ip_tuple *id) 133 | { 134 | struct bsys_desc *d = __bsys_arr_next(karr); 135 | ixev_check_hacks(ctx); 136 | 137 | ksys_tcp_connect(d, id, (unsigned long) ctx); 138 | } 139 | 140 | extern void ixev_ctx_init(struct ixev_ctx *ctx); 141 | extern void ixev_wait(void); 142 | 143 | extern void ixev_set_handler(struct ixev_ctx *ctx, unsigned int mask, 144 | ixev_handler_t handler); 145 | 146 | extern int ixev_init_thread(void); 147 | extern int ixev_init(struct ixev_conn_ops *ops); 148 | 149 | -------------------------------------------------------------------------------- /libix/ixev_timer.c: -------------------------------------------------------------------------------- 1 | #include "ixev_timer.h" 2 | #include "syscall.h" 3 | 4 | int ixev_timer_init(struct ixev_timer *t, ixev_timer_handler_t h, void *arg) 5 | { 6 | t->handler = h; 7 | t->arg = arg; 8 | t->timer_id = sys_timer_init(t); 9 | 10 | return t->timer_id != -1; 11 | } 12 | 13 | int ixev_timer_add(struct ixev_timer *t, struct timeval tv) 14 | { 15 | uint64_t delay; 16 | 17 | delay = tv.tv_sec * 1000000 + tv.tv_usec; 18 | 19 | return sys_timer_ctl(t->timer_id, delay); 20 | } 21 | -------------------------------------------------------------------------------- /libix/ixev_timer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | typedef void (*ixev_timer_handler_t)(void *arg); 6 | 7 | struct ixev_timer { 8 | ixev_timer_handler_t handler; 9 | void *arg; 10 | int timer_id; 11 | }; 12 | 13 | int ixev_timer_init(struct ixev_timer *t, ixev_timer_handler_t h, void *arg); 14 | 15 | int ixev_timer_add(struct ixev_timer *t, struct timeval tv); 16 | -------------------------------------------------------------------------------- /libix/main.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #include "syscall.h" 31 | #include "ix.h" 32 | 33 | static __thread bsysfn_t usys_tbl[USYS_NR]; 34 | static __thread struct bsys_arr *uarr; 35 | 36 | __thread struct bsys_arr *karr; 37 | 38 | /** 39 | * ix_poll - flush pending commands and check for new commands 40 | * 41 | * Returns the number of new commands received. 42 | */ 43 | int ix_poll(void) 44 | { 45 | int ret; 46 | 47 | ret = sys_bpoll(karr->descs, karr->len); 48 | if (ret) { 49 | printf("libix: encountered a fatal memory fault\n"); 50 | exit(-1); 51 | } 52 | 53 | return uarr->len; 54 | } 55 | 56 | void ix_handle_events(void) 57 | { 58 | int i; 59 | 60 | for (i = 0; i < uarr->len; i++) { 61 | struct bsys_desc d = uarr->descs[i]; 62 | usys_tbl[d.sysnr](d.arga, d.argb, d.argc, d.argd); 63 | } 64 | } 65 | 66 | /** 67 | * ix_flush - send pending commands 68 | */ 69 | void ix_flush(void) 70 | { 71 | int ret; 72 | 73 | ret = sys_bcall(karr->descs, karr->len); 74 | if (ret) { 75 | printf("libix: encountered a fatal memory fault\n"); 76 | exit(-1); 77 | } 78 | 79 | karr->len = 0; 80 | } 81 | 82 | static void 83 | ix_default_udp_recv(void *addr, size_t len, struct ip_tuple *id) 84 | { 85 | ix_udp_recv_done(addr); 86 | } 87 | 88 | static void 89 | ix_default_tcp_knock(int handle, struct ip_tuple *id) 90 | { 91 | ix_tcp_reject(handle); 92 | } 93 | 94 | /** 95 | * ix_init - initializes libIX 96 | * @ops: user-provided event handlers 97 | * @batch_depth: the maximum number of outstanding requests to the kernel 98 | * 99 | * Returns 0 if successful, otherwise fail. 100 | */ 101 | int ix_init(struct ix_ops *ops, int batch_depth) 102 | { 103 | if (!ops) 104 | return -EINVAL; 105 | 106 | /* unpack the ops into a more efficient representation */ 107 | usys_tbl[USYS_UDP_RECV] = (bsysfn_t) ops->udp_recv; 108 | usys_tbl[USYS_UDP_SENT] = (bsysfn_t) ops->udp_sent; 109 | usys_tbl[USYS_TCP_CONNECTED] = (bsysfn_t) ops->tcp_connected; 110 | usys_tbl[USYS_TCP_KNOCK] = (bsysfn_t) ops->tcp_knock; 111 | usys_tbl[USYS_TCP_RECV] = (bsysfn_t) ops->tcp_recv; 112 | usys_tbl[USYS_TCP_SENT] = (bsysfn_t) ops->tcp_sent; 113 | usys_tbl[USYS_TCP_DEAD] = (bsysfn_t) ops->tcp_dead; 114 | usys_tbl[USYS_TIMER] = (bsysfn_t) ops->timer_event; 115 | 116 | /* provide sane defaults so we don't leak memory */ 117 | if (!ops->udp_recv) 118 | usys_tbl[USYS_UDP_RECV] = (bsysfn_t) ix_default_udp_recv; 119 | if (!ops->tcp_knock) 120 | usys_tbl[USYS_TCP_KNOCK] = (bsysfn_t) ix_default_tcp_knock; 121 | 122 | uarr = sys_baddr(); 123 | if (!uarr) 124 | return -EFAULT; 125 | 126 | karr = malloc(sizeof(struct bsys_arr) + 127 | sizeof(struct bsys_desc) * batch_depth); 128 | if (!karr) 129 | return -ENOMEM; 130 | 131 | karr->len = 0; 132 | karr->max_len = batch_depth; 133 | 134 | return 0; 135 | } 136 | 137 | -------------------------------------------------------------------------------- /libix/mem.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * mem.c - memory management routines 26 | * 27 | * FIXME: this is really only a placeholder right now. 28 | */ 29 | 30 | #include 31 | #include 32 | #include "ix.h" 33 | 34 | static uintptr_t ixmem_pos = MEM_USER_IOMAPM_BASE_ADDR; 35 | static pthread_mutex_t ixmem_mutex = PTHREAD_MUTEX_INITIALIZER; 36 | 37 | /** 38 | * ixmem_alloc_pages - allocates 2MB pages 39 | * @nrpages: the number of pages 40 | * 41 | * Returns a start address, or NULL if fail. 42 | */ 43 | void *ix_alloc_pages(int nrpages) 44 | { 45 | void *addr; 46 | int ret; 47 | 48 | pthread_mutex_lock(&ixmem_mutex); 49 | addr = (void *) ixmem_pos; 50 | ixmem_pos += PGSIZE_2MB * nrpages; 51 | pthread_mutex_unlock(&ixmem_mutex); 52 | 53 | ret = sys_mmap(addr, nrpages, PGSIZE_2MB, VM_PERM_R | VM_PERM_W); 54 | if (ret) 55 | return NULL; 56 | 57 | return addr; 58 | } 59 | 60 | /** 61 | * ixmem_free_pages - frees 2MB pages 62 | * @addr: the start address 63 | * @nrpages: the number of pages 64 | */ 65 | void ix_free_pages(void *addr, int nrpages) 66 | { 67 | sys_unmap(addr, nrpages, PGSIZE_2MB); 68 | } 69 | 70 | -------------------------------------------------------------------------------- /libix/mempool.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * mempool.h - a fast fixed-sized memory pool allocator 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | #include 32 | #include 33 | 34 | #ifdef __KERNEL__ 35 | #error "wrong include ... this is userlevel mempool" 36 | #endif /* __KERNEL__ */ 37 | 38 | #define MEMPOOL_DEFAULT_CHUNKSIZE 128 39 | #define MEMPOOL_INITIAL_OFFSET (0) 40 | 41 | struct mempool_hdr { 42 | struct mempool_hdr *next; 43 | struct mempool_hdr *next_chunk; 44 | } __packed; 45 | 46 | 47 | // one per data type 48 | struct mempool_datastore { 49 | uint64_t magic; 50 | spinlock_t lock; 51 | struct mempool_hdr *chunk_head; 52 | void *buf; 53 | int nr_pages; 54 | uint32_t nr_elems; 55 | size_t elem_len; 56 | int nostraddle; 57 | int chunk_size; 58 | int num_chunks; 59 | int free_chunks; 60 | int64_t num_locks; 61 | const char *prettyname; 62 | struct mempool_datastore *next_ds; 63 | }; 64 | 65 | struct mempool { 66 | uint64_t magic; 67 | void *buf; 68 | struct mempool_datastore *datastore; 69 | struct mempool_hdr *head; 70 | struct mempool_hdr *private_chunk; 71 | // int nr_pages; 72 | uint32_t nr_elems; 73 | size_t elem_len; 74 | int nostraddle; 75 | int chunk_size; 76 | int num_alloc; 77 | int num_free; 78 | }; 79 | #define MEMPOOL_MAGIC 0x12911776 80 | 81 | /** 82 | * mempool_alloc - allocates an element from a memory pool 83 | * @m: the memory pool 84 | * 85 | * Returns a pointer to the allocated element or NULL if unsuccessful. 86 | */ 87 | extern void *mempool_alloc_2(struct mempool *m); 88 | static inline void *mempool_alloc(struct mempool *m) 89 | { 90 | struct mempool_hdr *h = m->head; 91 | 92 | if (likely(h)) { 93 | m->head = h->next; 94 | m->num_alloc++; 95 | m->num_free--; 96 | return (void *) h; 97 | } else { 98 | return mempool_alloc_2(m); 99 | } 100 | } 101 | 102 | /** 103 | * mempool_free - frees an element back in to a memory pool 104 | * @m: the memory pool 105 | * @ptr: the element 106 | * 107 | * NOTE: Must be the same memory pool that it was allocated from 108 | */ 109 | extern void mempool_free_2(struct mempool *m, void *ptr); 110 | static inline void mempool_free(struct mempool *m, void *ptr) 111 | { 112 | struct mempool_hdr *elem = (struct mempool_hdr *) ptr; 113 | 114 | if (likely(m->num_free < m->chunk_size)) { 115 | m->num_free++; 116 | m->num_alloc--; 117 | elem->next = m->head; 118 | m->head = elem; 119 | } else 120 | mempool_free_2(m, ptr); 121 | } 122 | 123 | extern int mempool_create_datastore(struct mempool_datastore *m, int nr_elems, size_t elem_len, int nostraddle, int chunk_size, const char *prettyname); 124 | extern int mempool_create(struct mempool *m, struct mempool_datastore *mds); 125 | extern void mempool_destroy(struct mempool *m); 126 | -------------------------------------------------------------------------------- /libix/syscall.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013-16 Board of Trustees of Stanford University 3 | * Copyright 2013-16 Ecole Polytechnique Federale Lausanne (EPFL) 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to deal 7 | * in the Software without restriction, including without limitation the rights 8 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | * THE SOFTWARE. 22 | */ 23 | 24 | /* 25 | * syscall.h - system call support 26 | */ 27 | 28 | #pragma once 29 | 30 | #include 31 | #include 32 | #include 33 | #include "syscall_raw.h" 34 | 35 | static inline int sys_bpoll(struct bsys_desc *d, unsigned int nr) 36 | { 37 | return (int) SYSCALL(SYS_BPOLL, d, nr); 38 | } 39 | 40 | static inline int sys_bcall(struct bsys_desc *d, unsigned int nr) 41 | { 42 | return (int) SYSCALL(SYS_BCALL, d, nr); 43 | } 44 | 45 | static inline void *sys_baddr(void) 46 | { 47 | return (struct bsys_arr *) SYSCALL(SYS_BADDR); 48 | } 49 | 50 | static inline int sys_mmap(void *addr, int nr, int size, int perm) 51 | { 52 | return (int) SYSCALL(SYS_MMAP, addr, nr, size, perm); 53 | } 54 | 55 | static inline int sys_unmap(void *addr, int nr, int size) 56 | { 57 | return (int) SYSCALL(SYS_MUNMAP, addr, nr, size); 58 | } 59 | 60 | static inline int sys_spawnmode(bool spawn_cores) 61 | { 62 | return (int) SYSCALL(SYS_SPAWNMODE, spawn_cores); 63 | } 64 | 65 | static inline int sys_nrcpus(void) 66 | { 67 | return (int) SYSCALL(SYS_NRCPUS); 68 | } 69 | 70 | static inline int sys_timer_init(void * addr) 71 | { 72 | return (int) SYSCALL(SYS_TIMER_INIT, addr); 73 | } 74 | 75 | static inline int sys_timer_ctl(int timer_id, uint64_t delay) 76 | { 77 | return (int) SYSCALL(SYS_TIMER_CTL, timer_id, delay); 78 | } 79 | -------------------------------------------------------------------------------- /tools/Makefile: -------------------------------------------------------------------------------- 1 | CFLAGS=-Wall -g -MD -O3 -I../inc 2 | LDFLAGS=-lrt 3 | 4 | ix-stats-show: ix-stats-show.o 5 | $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) 6 | 7 | clean: 8 | rm -f ix-stats-show *.o *.d 9 | 10 | .PHONY: clean 11 | 12 | -include *.d 13 | -------------------------------------------------------------------------------- /tools/ix-stats-show.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include 15 | #include 16 | 17 | // static void show_stat(const char *name, const char *fmt, ...) __attribute__ ((format (printf, 2, 3))); 18 | 19 | // static void show_stat(const char *name, const char *fmt, ...) 20 | // { 21 | // va_list ap; 22 | 23 | // printf("%s ", name); 24 | // va_start(ap, fmt); 25 | // vprintf(fmt, ap); 26 | // va_end(ap); 27 | // puts(""); 28 | // } 29 | 30 | #if !CONFIG_STATS 31 | 32 | int main(int argc, char **argv) 33 | { 34 | fprintf(stderr, "%s: Error: CONFIG_STATS was disabled during compilation.\n", argv[0]); 35 | return 1; 36 | } 37 | 38 | #else 39 | 40 | #define cpu_relax() asm volatile("pause") 41 | 42 | static void __attribute__((unused)) show_histogram(const char *name, int min, int max, int buckets, long sum, unsigned int *bucket_arr) 43 | { 44 | int i, from, to; 45 | long count = 0; 46 | double perc; 47 | 48 | /* prevent bug in gcc 5.4 https://godbolt.org/g/Mhfv83 */ 49 | unsigned int copy[256]; 50 | assert(buckets < sizeof(copy)/sizeof(copy[0])); 51 | memcpy(copy, bucket_arr, buckets * sizeof(int)); 52 | 53 | for (i = 0; i < buckets; i++) 54 | count += (long) copy[i]; 55 | 56 | printf("%s count %ld avg %.1f ", name, count, 1.0 * sum / count); 57 | for (i = 0; i < buckets; i++) { 58 | from = (i * (max - min) + buckets - 1) / buckets + min; 59 | to = ((i + 1) * (max - min) + buckets - 1) / buckets + min; 60 | perc = 100.0 * bucket_arr[i] / count; 61 | if (i == 0) 62 | printf("(-inf-%d): %.1f%% ", to, perc); 63 | else if (i == buckets - 1) 64 | printf("[%d-inf): %.1f%% ", from, perc); 65 | else 66 | printf("[%d-%d): %.1f%% ", from, to, perc); 67 | } 68 | puts(""); 69 | } 70 | 71 | static void __attribute__((unused)) show_counter(const char *name, int value) 72 | { 73 | printf("%s count %d\n", name, value); 74 | } 75 | 76 | static void show_stats(struct ix_stats *stats) 77 | { 78 | int i; 79 | struct ix_stats_percpu acc = {0}; 80 | 81 | for (i = 0; i < stats->cpus; i++) 82 | stats->percpu[i].poll = 1; 83 | 84 | for (i = 0; i < stats->cpus; i++) 85 | while (stats->percpu[i].poll) 86 | cpu_relax(); 87 | 88 | for (i = 0; i < stats->cpus; i++) { 89 | #define HISTOGRAM(name, min, max, buckets) \ 90 | acc.name.sum += stats->percpu[i].name.sum; \ 91 | for (int j = 0; j < buckets; j++) { \ 92 | acc.name.bucket[j] += stats->percpu[i].name.bucket[j]; \ 93 | } 94 | #define COUNTER(name) acc.name += stats->percpu[i].name; 95 | STATS 96 | #undef HISTOGRAM 97 | #undef COUNTER 98 | } 99 | 100 | #define HISTOGRAM(name, min, max, buckets) show_histogram(#name, min, max, buckets, acc.name.sum, acc.name.bucket); 101 | #define COUNTER(name) show_counter(#name, acc.name); 102 | STATS 103 | #undef HISTOGRAM 104 | } 105 | 106 | int main(int argc, char **argv) 107 | { 108 | int fd, i; 109 | int reset = 0; 110 | struct ix_stats *stats; 111 | 112 | if (argc > 1 && !strcmp(argv[1], "--reset")) 113 | reset = 1; 114 | 115 | fd = shm_open("/ix-stats", O_RDWR, 0); 116 | if (fd == -1) { 117 | perror("shm_open"); 118 | exit(1); 119 | } 120 | 121 | stats = mmap(NULL, sizeof(struct ix_stats), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 122 | if (stats == MAP_FAILED) { 123 | perror("mmap"); 124 | exit(1); 125 | } 126 | 127 | // bzero(stats, sizeof(*stats)); 128 | // STATS_HISTOGRAM(ethqueue, -999); 129 | // for (int i = IX_STATS_HISTOGRAM_MIN; i < IX_STATS_HISTOGRAM_MAX; i++) { 130 | // printf("%d ", i); 131 | // STATS_HISTOGRAM(ethqueue, i); 132 | // show_stats(stats); 133 | // } 134 | // STATS_HISTOGRAM(ethqueue, 999); 135 | 136 | show_stats(stats); 137 | 138 | if (reset) 139 | for (i = 0; i < stats->cpus; i++) 140 | stats->percpu[i].reset = 1; 141 | 142 | return 0; 143 | } 144 | 145 | #endif /* CONFIG_STATS */ 146 | --------------------------------------------------------------------------------