├── WORKSPACE ├── .gitignore ├── Makefile ├── src ├── BUILD ├── workqueue.h ├── workqueue.c └── server.c ├── .bazelrc ├── toolchain ├── BUILD └── cc_toolchain_config.bzl ├── Dockerfile ├── LICENSE.txt ├── README └── README.md /WORKSPACE: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.out 3 | tags 4 | bazel-* 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CC = gcc 2 | CFLAGS = -g -Wall 3 | OBJS = src/server.c src/workqueue.c 4 | LDFLAGS = -levent -lpthread 5 | TARGET = build/server.o 6 | 7 | all: $(TARGET) 8 | 9 | $(TARGET): $(OBJS) 10 | $(CC) $(OBJS) -o $@ $(LDFLAGS) 11 | 12 | clean: 13 | rm -f $(TARGET) 14 | -------------------------------------------------------------------------------- /src/BUILD: -------------------------------------------------------------------------------- 1 | load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") 2 | 3 | cc_library( 4 | name = "workqueue", 5 | srcs = ["workqueue.c"], 6 | hdrs = ["workqueue.h"], 7 | linkopts = ["-pthread", "-levent"], 8 | ) 9 | 10 | cc_binary( 11 | name = "server", 12 | srcs = ["server.c"], 13 | deps = [ 14 | ":workqueue", 15 | ], 16 | ) 17 | 18 | -------------------------------------------------------------------------------- /.bazelrc: -------------------------------------------------------------------------------- 1 | # Use our custom-configured c toolchain. 2 | 3 | build:clang_config --crosstool_top=//toolchain:clang_suite 4 | 5 | # Use --cpu as a differentiator. 6 | 7 | build:clang_config --cpu=k8 8 | 9 | # Use the default Bazel C toolchain to build the tools used during the 10 | # build. 11 | 12 | build:clang_config --host_crosstool_top=@bazel_tools//tools/cpp:toolchain 13 | 14 | # pg option for uftrace 15 | build:clang_config --cxxopt="-pg" 16 | -------------------------------------------------------------------------------- /toolchain/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | cc_toolchain_suite( 4 | name = "clang_suite", 5 | toolchains = { 6 | "k8": ":k8_toolchain", 7 | }, 8 | ) 9 | 10 | filegroup(name = "empty") 11 | 12 | cc_toolchain( 13 | name = "k8_toolchain", 14 | toolchain_identifier = "k8-toolchain", 15 | toolchain_config = ":k8_toolchain_config", 16 | all_files = ":empty", 17 | compiler_files = ":empty", 18 | dwp_files = ":empty", 19 | linker_files = ":empty", 20 | objcopy_files = ":empty", 21 | strip_files = ":empty", 22 | supports_param_files = 0, 23 | ) 24 | 25 | load(":cc_toolchain_config.bzl", "cc_toolchain_config") 26 | 27 | cc_toolchain_config(name = "k8_toolchain_config") 28 | -------------------------------------------------------------------------------- /src/workqueue.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Multithreaded work queue. 3 | * Copyright (c) 2012-2015 Ronald Bennett Cemer 4 | * This software is licensed under the BSD license. 5 | * See the accompanying LICENSE.txt for details. 6 | */ 7 | 8 | #ifndef WORKQUEUE_H 9 | #define WORKQUEUE_H 10 | 11 | #include 12 | 13 | typedef struct worker { 14 | pthread_t thread; 15 | int terminate; 16 | struct workqueue *workqueue; 17 | struct worker *prev; 18 | struct worker *next; 19 | } worker_t; 20 | 21 | typedef struct job { 22 | void (*job_function)(struct job *job); 23 | void *user_data; 24 | struct job *prev; 25 | struct job *next; 26 | } job_t; 27 | 28 | typedef struct workqueue { 29 | struct worker *workers; 30 | struct job *waiting_jobs; 31 | pthread_mutex_t jobs_mutex; 32 | pthread_cond_t jobs_cond; 33 | } workqueue_t; 34 | 35 | int workqueue_init(workqueue_t *workqueue, int numWorkers); 36 | 37 | void workqueue_shutdown(workqueue_t *workqueue); 38 | 39 | void workqueue_add_job(workqueue_t *workqueue, job_t *job); 40 | 41 | #endif /* #ifndef WORKQUEUE_H */ 42 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ######################### 2 | ######## alpine ######### 3 | ######################### 4 | FROM alpine:latest 5 | RUN apk add --no-cache --upgrade \ 6 | clang \ 7 | build-base \ 8 | libevent-dev 9 | 10 | ######################### 11 | ###### rockylinux ####### 12 | ######################### 13 | # FROM rockylinux/rockylinux:latest 14 | # RUN dnf install -y clang libevent-devel \ 15 | # && dnf clean all \ 16 | # && rm -rf /var/cache/yum 17 | 18 | ######################### 19 | ######## ubuntu ######### 20 | ######################### 21 | # FROM ubuntu:20.04 22 | # RUN apt update -y \ 23 | # && apt install -y clang \ 24 | # libevent-dev \ 25 | # && rm -rf /var/lib/apt/lists/* 26 | 27 | # If you want other distribute linux, then you check package *.so file version. 28 | # When *.o executed, then check lld which same *.so in binary file. 29 | 30 | WORKDIR /opt 31 | RUN mkdir src 32 | ADD src src 33 | 34 | RUN clang -o server.o src/server.c src/workqueue.c -levent -lpthread && \ 35 | rm -rf /opt/src && \ 36 | apk del clang \ 37 | build-base 38 | 39 | EXPOSE 8080 40 | 41 | # Keep container running for Test. 42 | # CMD exec /bin/sh -c "trap : TERM INT; sleep infinity & wait" 43 | ENTRYPOINT [ "./server.o" ] 44 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012, Ronald B. Cemer 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | Neither the name of Ronald B. Cemer nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 9 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 10 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Multithreaded, libevent-based socket server. 2 | 3 | Copyright (c) 2012 Ronald Bennett Cemer 4 | This software is licensed under the BSD license. 5 | See the accompanying LICENSE.txt for details. 6 | 7 | To compile: gcc -o echoserver_threaded echoserver_threaded.c workqueue.c -levent -lpthread 8 | 9 | To run: ./echoserver_threaded 10 | 11 | 12 | Libevent is a nice library for handling and dispatching events, as well as doing nonblocking I/O. This is fine, except that it is basically single-threaded -- which means that if you have multiple CPUs or a CPU with hyperthreading, you're really under-utilizing the CPU resources available to your server application because your event pump is running in a single thread and therefore can only use one CPU core at a time. 13 | 14 | The solution is to create one libevent event queues (AKA event_base) per active connection, each with its own event pump thread. This project does exactly that, giving you everything you need to write high-performance, multi-threaded, libevent-based socket servers. 15 | 16 | There are mentionings of running libevent in a multithreaded implementation, however it is very difficult (if not impossible) to find working implementations. This project is a working implementation of a multi-threaded, libevent-based socket server. 17 | 18 | The server itself simply echoes whatever you send to it. Start it up, then telnet to it: 19 | telnet localhost 5555 20 | Everything you type should be echoed back to you. 21 | 22 | The implementation is fairly standard. The main thread listens on a socket and accepts new connections, then farms the actual handling of those connections out to a pool of worker threads. Each connection has its own isolated event queue. 23 | 24 | In theory, for maximum performance, the number of worker threads should be set to the number of CPU cores available. Feel free to experiment with this. 25 | 26 | Also note that the server includes a multithreaded work queue implementation, which can be re-used for other purposes. 27 | 28 | Since the code is BSD licensed, you are free to use the source code however you wish, either in whole or in part. 29 | 30 | 31 | 32 | Some inspiration and coding ideas came from echoserver and cliserver, both of which are single-threaded, libevent-based servers. 33 | 34 | Echoserver is located here: http://ishbits.googlecode.com/svn/trunk/libevent-examples/echo-server/libevent_echosrv1.c 35 | Cliserver is located here: http://nitrogen.posterous.com/cliserver-an-example-libevent-based-socket-se 36 | -------------------------------------------------------------------------------- /toolchain/cc_toolchain_config.bzl: -------------------------------------------------------------------------------- 1 | # NEW 2 | load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") 3 | # NEW 4 | load( 5 | "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", 6 | "feature", 7 | "flag_group", 8 | "flag_set", 9 | "tool_path", 10 | ) 11 | 12 | all_link_actions = [ # NEW 13 | ACTION_NAMES.c_link_executable, 14 | ACTION_NAMES.c_link_dynamic_library, 15 | ACTION_NAMES.c_link_nodeps_dynamic_library, 16 | ] 17 | 18 | def _impl(ctx): 19 | tool_paths = [ 20 | tool_path( 21 | name = "gcc", 22 | path = "/usr/bin/clang-12", 23 | ), 24 | tool_path( 25 | name = "ld", 26 | path = "/usr/bin/ld", 27 | ), 28 | tool_path( 29 | name = "ar", 30 | path = "/bin/false", 31 | ), 32 | tool_path( 33 | name = "cpp", 34 | path = "/bin/false", 35 | ), 36 | tool_path( 37 | name = "gcov", 38 | path = "/bin/false", 39 | ), 40 | tool_path( 41 | name = "nm", 42 | path = "/bin/false", 43 | ), 44 | tool_path( 45 | name = "objdump", 46 | path = "/bin/false", 47 | ), 48 | tool_path( 49 | name = "strip", 50 | path = "/bin/false", 51 | ), 52 | ] 53 | 54 | # NEW 55 | features = [ 56 | feature( 57 | name = "default_linker_flags", 58 | enabled = True, 59 | flag_sets = [ 60 | flag_set( 61 | actions = all_link_actions, 62 | flag_groups = ([ 63 | flag_group( 64 | flags = [ 65 | 66 | ], 67 | ), 68 | ]), 69 | ), 70 | ], 71 | ), 72 | ] 73 | 74 | return cc_common.create_cc_toolchain_config_info( 75 | ctx = ctx, 76 | features = features, # NEW 77 | cxx_builtin_include_directories = [ 78 | "/usr/lib/llvm-12/lib/clang/12.0.0/include", 79 | "/usr/include" 80 | ], 81 | toolchain_identifier = "k8-toolchain", 82 | host_system_name = "local", 83 | target_system_name = "local", 84 | target_cpu = "k8", 85 | target_libc = "unknown", 86 | compiler = "clang", 87 | abi_version = "unknown", 88 | abi_libc_version = "unknown", 89 | tool_paths = tool_paths, 90 | ) 91 | 92 | cc_toolchain_config = rule( 93 | implementation = _impl, 94 | attrs = {}, 95 | provides = [CcToolchainConfigInfo], 96 | ) 97 | -------------------------------------------------------------------------------- /src/workqueue.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Multithreaded work queue. 3 | * Copyright (c) 2012-2015 Ronald Bennett Cemer 4 | * This software is licensed under the BSD license. 5 | * See the accompanying LICENSE.txt for details. 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include "workqueue.h" 12 | 13 | #define LL_ADD(item, list) \ 14 | do { \ 15 | item->prev = NULL; \ 16 | item->next = list; \ 17 | list = item; \ 18 | } while (0) 19 | 20 | #define LL_REMOVE(item, list) \ 21 | do { \ 22 | if (item->prev != NULL) item->prev->next = item->next; \ 23 | if (item->next != NULL) item->next->prev = item->prev; \ 24 | if (list == item) list = item->next; \ 25 | item->prev = item->next = NULL; \ 26 | } while (0) 27 | 28 | static void *worker_function(void *ptr) { 29 | worker_t *worker = (worker_t *)ptr; 30 | job_t *job; 31 | 32 | while (1) { 33 | /* Wait until we get notified. */ 34 | pthread_mutex_lock(&worker->workqueue->jobs_mutex); 35 | while (worker->workqueue->waiting_jobs == NULL) { 36 | /* If we're supposed to terminate, break out of our continuous loop. */ 37 | if (worker->terminate) break; 38 | 39 | pthread_cond_wait(&worker->workqueue->jobs_cond, &worker->workqueue->jobs_mutex); 40 | } 41 | 42 | /* If we're supposed to terminate, break out of our continuous loop. */ 43 | if (worker->terminate) { 44 | pthread_mutex_unlock(&worker->workqueue->jobs_mutex); 45 | break; 46 | } 47 | 48 | job = worker->workqueue->waiting_jobs; 49 | if (job != NULL) { 50 | LL_REMOVE(job, worker->workqueue->waiting_jobs); 51 | } 52 | pthread_mutex_unlock(&worker->workqueue->jobs_mutex); 53 | 54 | /* If we didn't get a job, then there's nothing to do at this time. */ 55 | if (job == NULL) continue; 56 | 57 | /* Execute the job. */ 58 | job->job_function(job); 59 | } 60 | 61 | free(worker); 62 | pthread_exit(NULL); 63 | } 64 | 65 | int workqueue_init(workqueue_t *workqueue, int numWorkers) { 66 | int i; 67 | worker_t *worker; 68 | pthread_cond_t blank_cond = PTHREAD_COND_INITIALIZER; 69 | pthread_mutex_t blank_mutex = PTHREAD_MUTEX_INITIALIZER; 70 | 71 | if (numWorkers < 1) numWorkers = 1; 72 | memset(workqueue, 0, sizeof(*workqueue)); 73 | memcpy(&workqueue->jobs_mutex, &blank_mutex, sizeof(workqueue->jobs_mutex)); 74 | memcpy(&workqueue->jobs_cond, &blank_cond, sizeof(workqueue->jobs_cond)); 75 | 76 | for (i = 0; i < numWorkers; i++) { 77 | if ((worker = malloc(sizeof(worker_t))) == NULL) { 78 | perror("Failed to allocate all workers"); 79 | return 1; 80 | } 81 | memset(worker, 0, sizeof(*worker)); 82 | worker->workqueue = workqueue; 83 | if (pthread_create(&worker->thread, NULL, worker_function, (void *)worker)) { 84 | perror("Failed to start all worker threads"); 85 | free(worker); 86 | return 1; 87 | } 88 | LL_ADD(worker, worker->workqueue->workers); 89 | } 90 | 91 | return 0; 92 | } 93 | 94 | void workqueue_shutdown(workqueue_t *workqueue) { 95 | worker_t *worker = NULL; 96 | 97 | /* Set all workers to terminate. */ 98 | for (worker = workqueue->workers; worker != NULL; worker = worker->next) { 99 | worker->terminate = 1; 100 | } 101 | 102 | /* Remove all workers and jobs from the work queue. 103 | * wake up all workers so that they will terminate. */ 104 | pthread_mutex_lock(&workqueue->jobs_mutex); 105 | workqueue->workers = NULL; 106 | workqueue->waiting_jobs = NULL; 107 | pthread_cond_broadcast(&workqueue->jobs_cond); 108 | pthread_mutex_unlock(&workqueue->jobs_mutex); 109 | } 110 | 111 | void workqueue_add_job(workqueue_t *workqueue, job_t *job) { 112 | /* Add the job to the job queue, and notify a worker. */ 113 | pthread_mutex_lock(&workqueue->jobs_mutex); 114 | LL_ADD(job, workqueue->waiting_jobs); 115 | pthread_cond_signal(&workqueue->jobs_cond); 116 | pthread_mutex_unlock(&workqueue->jobs_mutex); 117 | } 118 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Multithreaded, libevent-based socket server. 2 | 3 | ## Using nice libevent 4 | 5 | Libevent is a nice library for handling and dispatching events, as well as doing nonblocking I/O. 6 | 7 | This is fine, except that it is basically single-threaded. 8 | 9 | ## Do not under-utilize your computing resource 10 | 11 | If you have multiple CPUs or a CPU with hyperthreading, 12 | 13 | you're really under-utilizing the CPU resources available to your server application. 14 | 15 | Because your event pump is running in a single thread and therefore can only use one CPU core at a time. 16 | 17 | ## Event Queueing 18 | 19 | The solution is to create one libevent event queues (AKA event_base) per active connection, 20 | 21 | each with its own event pump thread. 22 | 23 | This project does exactly that, 24 | 25 | giving you everything you need to write high-performance, multi-threaded, libevent-based socket servers. 26 | 27 | There are mentionings of running libevent in a multithreaded implementation, 28 | 29 | however it is very difficult (if not impossible) to find working implementations. 30 | 31 | This project is a working implementation of a multi-threaded, libevent-based socket server. 32 | 33 | # Build 34 | 35 | Build output `server.o` is in `build` directory. 36 | 37 | You can choose each build method. 38 | 39 | ## Build with bazel 40 | 41 | Bazel look up `src/BUILD` file. 42 | 43 | ```console 44 | foo@bar:~/multithread-libevent-echo-server$ bazel build //src:server 45 | INFO: Analyzed target //src:server (0 packages loaded, 0 targets configured). 46 | INFO: Found 1 target... 47 | Target //src:server up-to-date: 48 | bazel-bin/src/server 49 | INFO: Elapsed time: 0.121s, Critical Path: 0.00s 50 | INFO: 1 process: 1 internal. 51 | INFO: Build completed successfully, 1 total action 52 | ``` 53 | 54 | Then, copy executable file. 55 | 56 | ```console 57 | foo@bar:~/multithread-libevent-echo-server$ cp bazel-bin/src/server build/server.o 58 | ``` 59 | 60 | ## Build with Make 61 | 62 | ```console 63 | foo@bar:~/multithread-libevent-echo-server$ make 64 | ``` 65 | 66 | ## Using native CC 67 | 68 | Go to src directory. 69 | 70 | ```console 71 | foo@bar:~/multithread-libevent-echo-server$ cd src 72 | ``` 73 | 74 | ### GCC 75 | 76 | ```console 77 | foo@bar:~/multithread-libevent-echo-server$ gcc -o ../build/server.o src/server.c src/workqueue.c -levent -lpthread 78 | ``` 79 | 80 | ### clang 81 | 82 | ```console 83 | foo@bar:~/multithread-libevent-echo-server$ clang-12 -o ../build/server.o src/server.c src/workqueue.c -levent -lpthread 84 | ``` 85 | 86 | # Test Echo 87 | 88 | ## Run Server 89 | 90 | Server running in port defined in server.c `[SERVER_PORT]`. 91 | 92 | ### Host server 93 | 94 | ```console 95 | foo@bar:~/multithread-libevent-echo-server/build$./server.o 96 | Server running in [SERVER_PORT] 97 | ``` 98 | 99 | ### Run on Docker 100 | 101 | Build Docker image. 102 | 103 | ```console 104 | foo@bar:~/multithread-libevent-echo-server/$ podman build -t multithread-event-server . 105 | ``` 106 | 107 | Run container. 108 | 109 | ```console 110 | foo@bar:~/multithread-libevent-echo-server/$ podman run -p 8080:8080 --name multithread-event-server multithread-event-server:latest 111 | Server running in [SERVER_PORT] 112 | ``` 113 | 114 | ## Run on host client 115 | 116 | The server itself simply echoes whatever you send to it. 117 | 118 | Start it up, then telnet 119 | 120 | ```console 121 | foo@bar:~/telnet localhost 8080 122 | ``` 123 | 124 | # License 125 | 126 | Copyright (c) 2012 Ronald Bennett Cemer 127 | 128 | This software is licensed under the BSD license. 129 | 130 | See the accompanying LICENSE.txt for details. 131 | 132 | # References 133 | 134 | [Echoserver link](http://ishbits.googlecode.com/svn/trunk/libevent-examples/echo-server/libevent_echosrv1.c) 135 | 136 | [Cliserver link](http://nitrogen.posterous.com/cliserver-an-example-libevent-based-socket-se) 137 | 138 | [Bazel Build System with C](https://github.com/research-note/bazel-clang-c-example) 139 | -------------------------------------------------------------------------------- /src/server.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Multithreaded, libevent-based socket server. 3 | * Copyright (c) 2012-2015 Ronald Bennett Cemer 4 | * This software is licensed under the BSD license. 5 | * See the accompanying LICENSE.txt for details. 6 | * 7 | * Maintained by 8 | * Paran Lee 9 | * 10 | * To compile: 11 | * gcc -o server.o server.c workqueue.c -levent -lpthread 12 | * To run: 13 | * ./server.o 14 | */ 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | #include "workqueue.h" 32 | 33 | /* Port to listen on. */ 34 | #define DEFULT_SERVER_PORT 8080 35 | 36 | /* Connection backlog (# of backlogged connections to accept). */ 37 | #define CONNECTION_BACKLOG 8 38 | 39 | /* Socket read and write timeouts, in seconds. */ 40 | #define SOCKET_READ_TIMEOUT_SECONDS 10 41 | #define SOCKET_WRITE_TIMEOUT_SECONDS 10 42 | 43 | /** 44 | * Behaves similarly to fprintf(stderr, ...), but adds file, line, and function 45 | * information. 46 | */ 47 | #define errorOut(...) \ 48 | do { \ 49 | fprintf(stderr, "%s:%d: %s():\t", __FILE__, __LINE__, __FUNCTION__); \ 50 | fprintf(stderr, __VA_ARGS__); \ 51 | } while(0) 52 | 53 | /** 54 | * Struct to carry around connection (client)-specific data. 55 | */ 56 | typedef struct client { 57 | /* The client's socket. */ 58 | int fd; 59 | 60 | /* The event_base for this client. */ 61 | struct event_base *evbase; 62 | 63 | /* The bufferedevent for this client. */ 64 | struct bufferevent *buf_ev; 65 | 66 | /* The output buffer for this client. */ 67 | struct evbuffer *output_buffer; 68 | 69 | /* Here you can add your own application-specific attributes which 70 | * are connection-specific. */ 71 | } client_t; 72 | 73 | static struct event_base *evbase_accept; 74 | static workqueue_t workqueue; 75 | 76 | /* Signal handler function (defined below). */ 77 | static void sighandler(int signal); 78 | 79 | /** 80 | * Set a socket to non-blocking mode. 81 | */ 82 | static int setnonblock(int fd) { 83 | int flags; 84 | 85 | flags = fcntl(fd, F_GETFL); 86 | if (flags < 0) return flags; 87 | flags |= O_NONBLOCK; 88 | if (fcntl(fd, F_SETFL, flags) < 0) return -1; 89 | return 0; 90 | } 91 | 92 | static void closeClient(client_t *client) { 93 | if (client != NULL) { 94 | if (client->fd >= 0) { 95 | close(client->fd); 96 | client->fd = -1; 97 | } 98 | } 99 | } 100 | 101 | static void closeAndFreeClient(client_t *client) { 102 | if (client != NULL) { 103 | closeClient(client); 104 | if (client->buf_ev != NULL) { 105 | bufferevent_free(client->buf_ev); 106 | client->buf_ev = NULL; 107 | } 108 | if (client->evbase != NULL) { 109 | event_base_free(client->evbase); 110 | client->evbase = NULL; 111 | } 112 | if (client->output_buffer != NULL) { 113 | evbuffer_free(client->output_buffer); 114 | client->output_buffer = NULL; 115 | } 116 | free(client); 117 | } 118 | } 119 | 120 | /** 121 | * Called by libevent when there is data to read. 122 | */ 123 | void buffered_on_read(struct bufferevent *bev, void *arg) { 124 | client_t *client = (client_t *)arg; 125 | char data[4096]; 126 | int nbytes; 127 | 128 | /* Copy the data from the input buffer to the output buffer in 4096-byte chunks. 129 | * There is a one-liner to do the whole thing in one shot, but the purpose of this server 130 | * is to show actual real-world reading and writing of the input and output buffers, 131 | * so we won't take that shortcut here. */ 132 | while ((nbytes = EVBUFFER_LENGTH(bev->input)) > 0) { 133 | /* Remove a chunk of data from the input buffer, copying it into our local array (data). */ 134 | if (nbytes > 4096) nbytes = 4096; 135 | evbuffer_remove(bev->input, data, nbytes); 136 | 137 | printf("client [%d]: %s", client->fd, data); 138 | /* Add the chunk of data from our local array (data) to the client's output buffer. */ 139 | evbuffer_add(client->output_buffer, data, nbytes); 140 | } 141 | 142 | /* Send the results to the client. This actually only queues the results for sending. 143 | * Sending will occur asynchronously, handled by libevent. */ 144 | if (bufferevent_write_buffer(bev, client->output_buffer)) { 145 | errorOut("Error sending data to client on fd %d\n", client->fd); 146 | closeClient(client); 147 | } 148 | } 149 | 150 | /** 151 | * Called by libevent when the write buffer reaches 0. We only 152 | * provide this because libevent expects it, but we don't use it. 153 | */ 154 | void buffered_on_write(struct bufferevent *bev, void *arg) { 155 | } 156 | 157 | /** 158 | * Called by libevent when there is an error on the underlying socket 159 | * descriptor. 160 | */ 161 | void buffered_on_error(struct bufferevent *bev, short what, void *arg) { 162 | closeClient((client_t *)arg); 163 | } 164 | 165 | static void server_job_function(struct job *job) { 166 | client_t *client = (client_t *)job->user_data; 167 | 168 | event_base_dispatch(client->evbase); 169 | closeAndFreeClient(client); 170 | free(job); 171 | } 172 | 173 | /** 174 | * This function will be called by libevent when there is a connection 175 | * ready to be accepted. 176 | */ 177 | void on_accept(int fd, short ev, void *arg) { 178 | int client_fd; 179 | struct sockaddr_in client_addr; 180 | socklen_t client_len = sizeof(client_addr); 181 | workqueue_t *workqueue = (workqueue_t *)arg; 182 | client_t *client; 183 | job_t *job; 184 | 185 | client_fd = accept(fd, (struct sockaddr *)&client_addr, &client_len); 186 | if (client_fd < 0) { 187 | warn("accept failed"); 188 | return; 189 | } 190 | 191 | /* Set the client socket to non-blocking mode. */ 192 | if (setnonblock(client_fd) < 0) { 193 | warn("failed to set client socket to non-blocking"); 194 | close(client_fd); 195 | return; 196 | } 197 | 198 | /* Create a client object. */ 199 | if ((client = malloc(sizeof(*client))) == NULL) { 200 | warn("failed to allocate memory for client state"); 201 | close(client_fd); 202 | return; 203 | } 204 | memset(client, 0, sizeof(*client)); 205 | client->fd = client_fd; 206 | 207 | /** 208 | * Add any custom code anywhere from here to the end of this function 209 | * to initialize your application-specific attributes in the client struct. 210 | **/ 211 | 212 | if ((client->output_buffer = evbuffer_new()) == NULL) { 213 | warn("client output buffer allocation failed"); 214 | closeAndFreeClient(client); 215 | return; 216 | } 217 | 218 | if ((client->evbase = event_base_new()) == NULL) { 219 | warn("client event_base creation failed"); 220 | closeAndFreeClient(client); 221 | return; 222 | } 223 | 224 | /** 225 | * 226 | * Create the buffered event. 227 | * 228 | * The first argument is the file descriptor that will trigger 229 | * the events, in this case the clients socket. 230 | * 231 | * The second argument is the callback that will be called 232 | * when data has been read from the socket and is available to 233 | * the application. 234 | * 235 | * The third argument is a callback to a function that will be 236 | * called when the write buffer has reached a low watermark. 237 | * That usually means that when the write buffer is 0 length, 238 | * this callback will be called. It must be defined, but you 239 | * don't actually have to do anything in this callback. 240 | * 241 | * The fourth argument is a callback that will be called when 242 | * there is a socket error. This is where you will detect 243 | * that the client disconnected or other socket errors. 244 | * 245 | * The fifth and final argument is to store an argument in 246 | * that will be passed to the callbacks. We store the client 247 | * object here. 248 | */ 249 | if ((client->buf_ev = bufferevent_new(client_fd, 250 | buffered_on_read, buffered_on_write, 251 | buffered_on_error, client)) == NULL) { 252 | warn("client bufferevent creation failed"); 253 | closeAndFreeClient(client); 254 | return; 255 | } 256 | bufferevent_base_set(client->evbase, client->buf_ev); 257 | 258 | bufferevent_settimeout(client->buf_ev, SOCKET_READ_TIMEOUT_SECONDS, 259 | SOCKET_WRITE_TIMEOUT_SECONDS); 260 | 261 | /* We have to enable it before our callbacks will be 262 | * called. */ 263 | bufferevent_enable(client->buf_ev, EV_READ); 264 | 265 | /* Create a job object and add it to the work queue. */ 266 | if ((job = malloc(sizeof(*job))) == NULL) { 267 | warn("failed to allocate memory for job state"); 268 | closeAndFreeClient(client); 269 | return; 270 | } 271 | job->job_function = server_job_function; 272 | job->user_data = client; 273 | 274 | workqueue_add_job(workqueue, job); 275 | } 276 | 277 | /** 278 | * Run the server. 279 | * This function blocks, only returning when the server has terminated. 280 | */ 281 | int runServer(int port) { 282 | int listenfd; 283 | struct sockaddr_in listen_addr; 284 | struct event ev_accept; 285 | int reuseaddr_on; 286 | 287 | /* Initialize libevent. */ 288 | event_init(); 289 | 290 | /* Set signal handlers */ 291 | sigset_t sigset; 292 | sigemptyset(&sigset); 293 | struct sigaction siginfo = { 294 | .sa_handler = sighandler, 295 | .sa_mask = sigset, 296 | .sa_flags = SA_RESTART, 297 | }; 298 | sigaction(SIGINT, &siginfo, NULL); 299 | sigaction(SIGTERM, &siginfo, NULL); 300 | 301 | /* Create our listening socket. */ 302 | listenfd = socket(AF_INET, SOCK_STREAM, 0); 303 | if (listenfd < 0) { 304 | err(1, "listen failed"); 305 | } 306 | memset(&listen_addr, 0, sizeof(listen_addr)); 307 | listen_addr.sin_family = AF_INET; 308 | listen_addr.sin_addr.s_addr = INADDR_ANY; 309 | listen_addr.sin_port = htons(port); 310 | if (bind(listenfd, (struct sockaddr *)&listen_addr, sizeof(listen_addr)) < 0) { 311 | err(1, "bind failed"); 312 | } 313 | if (listen(listenfd, CONNECTION_BACKLOG) < 0) { 314 | err(1, "listen failed"); 315 | } 316 | reuseaddr_on = 1; 317 | setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, 318 | &reuseaddr_on, sizeof(reuseaddr_on)); 319 | 320 | /* Set the socket to non-blocking, this is essential in event 321 | * based programming with libevent. */ 322 | if (setnonblock(listenfd) < 0) { 323 | err(1, "failed to set server socket to non-blocking"); 324 | } 325 | 326 | if ((evbase_accept = event_base_new()) == NULL) { 327 | perror("Unable to create socket accept event base"); 328 | close(listenfd); 329 | return 1; 330 | } 331 | 332 | int nrhart = get_nprocs(); 333 | printf("This system has %d processors configured and " 334 | "%d processors available.\n", 335 | get_nprocs_conf(), get_nprocs()); 336 | /** 337 | * Get the number of processors to execute worker threads. 338 | * this match number of CPU cores reported in /proc/cpuinfo 339 | * that currently available in the system. 340 | * 341 | * Initialize work queue. 342 | */ 343 | printf("Run with %d hardware threads.\n", nrhart); 344 | if (workqueue_init(&workqueue, nrhart)) { 345 | perror("Failed to create work queue"); 346 | close(listenfd); 347 | workqueue_shutdown(&workqueue); 348 | return 1; 349 | } 350 | 351 | /* We now have a listening socket, we create a read event to 352 | * be notified when a client connects. */ 353 | event_set(&ev_accept, listenfd, EV_READ|EV_PERSIST, 354 | on_accept, (void *)&workqueue); 355 | event_base_set(evbase_accept, &ev_accept); 356 | event_add(&ev_accept, NULL); 357 | 358 | printf("Server running in port %d.\n", port); 359 | 360 | /* Start the event loop. */ 361 | event_base_dispatch(evbase_accept); 362 | 363 | event_base_free(evbase_accept); 364 | evbase_accept = NULL; 365 | 366 | close(listenfd); 367 | 368 | printf("Server shutdown.\n"); 369 | 370 | return 0; 371 | } 372 | 373 | /** 374 | * Kill the server. This function can be called from another thread to kill the 375 | * server, causing runServer() to return. 376 | */ 377 | void killServer(void) { 378 | fprintf(stdout, "Stopping socket listener event loop.\n"); 379 | if (event_base_loopexit(evbase_accept, NULL)) { 380 | perror("Error shutting down server"); 381 | } 382 | 383 | fprintf(stdout, "Stopping workers.\n"); 384 | workqueue_shutdown(&workqueue); 385 | } 386 | 387 | static void sighandler(int signal) { 388 | fprintf(stdout, "Received signal %d: %s. Shutting down.\n", 389 | signal, strsignal(signal)); 390 | killServer(); 391 | } 392 | 393 | /** 394 | * Main function for demonstrating the echo server. 395 | * You can remove this and simply call runServer() from your application. 396 | */ 397 | int main(int argc, char *argv[]) { 398 | int port = argc > 1 && atoi(argv[1]) > 0 ? : DEFULT_SERVER_PORT; 399 | return runServer(port); 400 | } 401 | --------------------------------------------------------------------------------