├── README.md ├── acceptor.h ├── utils.h ├── utils.cpp ├── channel_map.h ├── log.h ├── event_loop_thread.h ├── makefile ├── event_loop_thread_pool.h ├── buffer.h ├── event_dispatcher.h ├── channel.h ├── acceptor.cpp ├── channel.cpp ├── channel_map.cpp ├── tcp_server.h ├── common.h ├── tcp_connection.h ├── event_loop_thread_pool.cpp ├── event_loop_thread.cpp ├── event_loop.h ├── epoll_server_multithreads.cpp ├── log.cpp ├── buffer.cpp ├── tcp_connection.cpp ├── event_dispatcher.cpp ├── tcp_server.cpp └── event_loop.cpp /README.md: -------------------------------------------------------------------------------- 1 | # epoll_server_multithreads 2 | 3 | 极客时间《网络编程》epoll 主从反应堆模式 部分的代码，将 epoll 主从反应堆模式 部分的代码单独剥离出来，并写了简单的makefile。 4 | -------------------------------------------------------------------------------- /acceptor.h: -------------------------------------------------------------------------------- 1 | #ifndef ACCEPTOR_H 2 | #define ACCEPTOR_H 3 | 4 | #include "common.h" 5 | 6 | struct acceptor{ 7 | int listen_port; 8 | int listen_fd; 9 | } ; 10 | 11 | struct acceptor *acceptor_init(int port); 12 | 13 | 14 | #endif -------------------------------------------------------------------------------- /utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_H 2 | #define UTILS_H 3 | 4 | #include "event_loop.h" 5 | 6 | void assertInSameThread(struct event_loop *eventLoop); 7 | 8 | //1： same thread: 0： not the same thread 9 | int isInSameThread(struct event_loop *eventLoop); 10 | #endif -------------------------------------------------------------------------------- /utils.cpp: -------------------------------------------------------------------------------- 1 | #include "utils.h" 2 | #include "log.h" 3 | 4 | 5 | void assertInSameThread(struct event_loop *eventLoop) { 6 | if (eventLoop->owner_thread_id != pthread_self()) { 7 | LOG_ERR("not in the same thread"); 8 | exit(-1); 9 | } 10 | } 11 | 12 | //1： same thread: 0： not the same thread 13 | int isInSameThread(struct event_loop *eventLoop){ 14 | return eventLoop->owner_thread_id == pthread_self(); 15 | } -------------------------------------------------------------------------------- /channel_map.h: -------------------------------------------------------------------------------- 1 | #ifndef CHANNEL_MAP_H 2 | #define CHANNEL_MAP_H 3 | 4 | 5 | #include "channel.h" 6 | 7 | /** 8 | * channel映射表, key为对应的socket描述字 9 | */ 10 | struct channel_map { 11 | void **entries; 12 | 13 | /* The number of entries available in entries */ 14 | int nentries; 15 | }; 16 | 17 | 18 | int map_make_space(struct channel_map *map, int slot, int msize); 19 | 20 | void map_init(struct channel_map *map); 21 | 22 | void map_clear(struct channel_map *map); 23 | 24 | #endif -------------------------------------------------------------------------------- /log.h: -------------------------------------------------------------------------------- 1 | #ifndef LOG_H 2 | #define LOG_H 3 | 4 | #include "stdarg.h" 5 | 6 | #define LOG_DEBUG_TYPE 0 7 | #define LOG_MSG_TYPE 1 8 | #define LOG_WARN_TYPE 2 9 | #define LOG_ERR_TYPE 3 10 | 11 | void yolanda_log(int severity, const char *msg); 12 | void yolanda_logx(int severity, const char *errstr, const char *fmt, va_list ap); 13 | void yolanda_msgx(const char *fmt, ...); 14 | void yolanda_debugx(const char *fmt, ...); 15 | 16 | 17 | #define LOG_MSG(msg) yolanda_log(LOG_MSG_TYPE, msg) 18 | #define LOG_ERR(msg) yolanda_log(LOG_ERR_TYPE, msg) 19 | 20 | void error(int status, int err, char *fmt, ...); 21 | 22 | #endif -------------------------------------------------------------------------------- /event_loop_thread.h: -------------------------------------------------------------------------------- 1 | #ifndef EVENT_LOOP_THREAD_H 2 | #define EVENT_LOOP_THREAD_H 3 | 4 | #include 5 | 6 | struct event_loop_thread { 7 | struct event_loop *eventLoop; 8 | pthread_t thread_tid; /* thread ID */ 9 | pthread_mutex_t mutex; 10 | pthread_cond_t cond; 11 | char * thread_name; 12 | long thread_count; /* # connections handled */ 13 | }; 14 | 15 | //初始化已经分配内存的event_loop_thread 16 | int event_loop_thread_init(struct event_loop_thread *, int); 17 | 18 | //由主线程调用，初始化一个子线程，并且让子线程开始运行event_loop 19 | struct event_loop *event_loop_thread_start(struct event_loop_thread *); 20 | 21 | #endif -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | TARGET := server 2 | 3 | # The C/C++ program compiler. 4 | CC=gcc 5 | 6 | # 编译器在编译时的参数设置,包含头文件路径设置 7 | CFLAGS:=-Wall -O2 -g 8 | CFLAGS+=-I $(shell pwd) 9 | 10 | # 库文件添加 11 | LDFLAGS:= 12 | LDFLAGS+=-lstdc++ -lpthread 13 | 14 | # 指定源程序存放位置 15 | SRCDIRS:=. 16 | 17 | # 设置程序中使用文件类型 18 | SRCEXTS:=.cpp 19 | 20 | # 设置运行程序名 21 | PROGRAM:=hello 22 | 23 | CFLAGS:=-Wall -O2 -g 24 | CFLAGS+=-I $(shell pwd) 25 | 26 | SOURCES=$(foreach d,$(SRCDIRS),$(wildcard $(addprefix $(d)/*,$(SRCEXTS)))) 27 | OBJS=$(foreach x,$(SRCEXTS),$(patsubst %$(x),%.o,$(filter %$(x),$(SOURCES)))) 28 | 29 | .PHONY: all clean 30 | 31 | %.o: %.cpp 32 | $(CC) -c $(CFLAGS) -o $@ $< 33 | 34 | $(PROGRAM): $(OBJS) 35 | $(CC) -o $(PROGRAM) $(OBJS) $(LDFLAGS) 36 | 37 | all: 38 | @echo $(OBJS) 39 | 40 | clean: 41 | rm -rf $(PROGRAM) *.o -------------------------------------------------------------------------------- /event_loop_thread_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef THREAD_POOL_H 2 | #define THREAD_POOL_H 3 | 4 | #include "event_loop.h" 5 | #include "event_loop_thread.h" 6 | 7 | struct event_loop_thread_pool { 8 | //创建thread_pool的主线程 9 | struct event_loop *mainLoop; 10 | //是否已经启动 11 | int started; 12 | //线程数目 13 | int thread_number; 14 | //数组指针，指向创建的event_loop_thread数组 15 | struct event_loop_thread *eventLoopThreads; 16 | 17 | //表示在数组里的位置，用来决定选择哪个event_loop_thread服务 18 | int position; 19 | 20 | }; 21 | 22 | struct event_loop_thread_pool *event_loop_thread_pool_new(struct event_loop *mainLoop, int threadNumber); 23 | 24 | void event_loop_thread_pool_start(struct event_loop_thread_pool *); 25 | 26 | struct event_loop *event_loop_thread_pool_get_loop(struct event_loop_thread_pool *); 27 | 28 | #endif -------------------------------------------------------------------------------- /buffer.h: -------------------------------------------------------------------------------- 1 | #ifndef BUFFER_H 2 | #define BUFFER_H 3 | 4 | #define INIT_BUFFER_SIZE 65536 5 | //数据缓冲区 6 | struct buffer { 7 | char *data; //实际缓冲 8 | int readIndex; //缓冲读取位置 9 | int writeIndex; //缓冲写入位置 10 | int total_size; //总大小 11 | }; 12 | 13 | 14 | struct buffer *buffer_new(); 15 | 16 | void buffer_free(struct buffer *buffer); 17 | 18 | int buffer_writeable_size(struct buffer *buffer); 19 | 20 | int buffer_readable_size(struct buffer *buffer); 21 | 22 | int buffer_front_spare_size(struct buffer *buffer); 23 | 24 | //往buffer里写数据 25 | int buffer_append(struct buffer *buffer, void *data, int size); 26 | 27 | //往buffer里写数据 28 | int buffer_append_char(struct buffer *buffer, char data); 29 | 30 | //往buffer里写数据 31 | int buffer_append_string(struct buffer*buffer, char * data); 32 | 33 | //读socket数据，往buffer里写 34 | int buffer_socket_read(struct buffer *buffer, int fd); 35 | 36 | //读buffer数据 37 | char buffer_read_char(struct buffer *buffer); 38 | 39 | //查询buffer数据 40 | char * buffer_find_CRLF(struct buffer * buffer); 41 | 42 | #endif -------------------------------------------------------------------------------- /event_dispatcher.h: -------------------------------------------------------------------------------- 1 | #ifndef EVENT_DISPATCHER_H 2 | #define EVENT_DISPATCHER_H 3 | 4 | #include "channel.h" 5 | #include "event_loop.h" 6 | 7 | /** 抽象的event_dispatcher结构体，对应的实现如select,poll,epoll等I/O复用. */ 8 | struct event_dispatcher { 9 | /** 对应实现 */ 10 | const char *name; 11 | 12 | /** 初始化函数 */ 13 | void *(*init)(struct event_loop * eventLoop); 14 | 15 | /** 通知dispatcher新增一个channel事件*/ 16 | int (*add)(struct event_loop * eventLoop, struct channel * channel); 17 | 18 | /** 通知dispatcher删除一个channel事件*/ 19 | int (*del)(struct event_loop * eventLoop, struct channel * channel); 20 | 21 | /** 通知dispatcher更新channel对应的事件*/ 22 | int (*update)(struct event_loop * eventLoop, struct channel * channel); 23 | 24 | /** 实现事件分发，然后调用event_loop的event_activate方法执行callback*/ 25 | int (*dispatch)(struct event_loop * eventLoop, struct timeval *); 26 | 27 | /** 清除数据 */ 28 | void (*clear)(struct event_loop * eventLoop); 29 | }; 30 | 31 | extern const struct event_dispatcher poll_dispatcher; 32 | extern const struct event_dispatcher epoll_dispatcher; 33 | 34 | #endif -------------------------------------------------------------------------------- /channel.h: -------------------------------------------------------------------------------- 1 | #ifndef CHANNEL_H 2 | #define CHANNEL_H 3 | 4 | #include 5 | 6 | #define EVENT_TIMEOUT 0x01 7 | /** Wait for a socket or FD to become readable */ 8 | #define EVENT_READ 0x02 9 | /** Wait for a socket or FD to become writeable */ 10 | #define EVENT_WRITE 0x04 11 | /** Wait for a POSIX signal to be raised*/ 12 | #define EVENT_SIGNAL 0x08 13 | 14 | 15 | typedef int (*event_read_callback)(void *data); 16 | 17 | typedef int (*event_write_callback)(void *data); 18 | 19 | struct channel { 20 | int fd; 21 | int events; //表示event类型 22 | 23 | event_read_callback eventReadCallback; 24 | event_write_callback eventWriteCallback; 25 | void *data; //callback data, 可能是event_loop，也可能是tcp_server或者tcp_connection 26 | }; 27 | 28 | 29 | struct channel *channel_new(int fd, int events, event_read_callback eventReadCallback, event_write_callback eventWriteCallback, void *data); 30 | 31 | int channel_write_event_is_enabled(struct channel *channel); 32 | 33 | int channel_write_event_enable(struct channel *channel); 34 | 35 | int channel_write_event_disable(struct channel *channel); 36 | 37 | 38 | #endif -------------------------------------------------------------------------------- /acceptor.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "acceptor.h" 3 | #include "tcp_server.h" 4 | #include "log.h" 5 | 6 | struct acceptor *acceptor_init(int port) { 7 | struct acceptor *acceptor1 = (struct acceptor *)malloc(sizeof(struct acceptor)); 8 | acceptor1->listen_port = port; 9 | acceptor1->listen_fd = socket(AF_INET, SOCK_STREAM, 0); 10 | 11 | make_nonblocking(acceptor1->listen_fd); 12 | 13 | struct sockaddr_in server_addr; 14 | bzero(&server_addr, sizeof(server_addr)); 15 | server_addr.sin_family = AF_INET; 16 | server_addr.sin_addr.s_addr = htonl(INADDR_ANY); 17 | server_addr.sin_port = htons(port); 18 | 19 | int on = 1; 20 | setsockopt(acceptor1->listen_fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)); 21 | 22 | int rt1 = bind(acceptor1->listen_fd, (struct sockaddr *) &server_addr, sizeof(server_addr)); 23 | if (rt1 < 0) { 24 | error(1, errno, "bind failed "); 25 | } 26 | 27 | int rt2 = listen(acceptor1->listen_fd, LISTENQ); 28 | if (rt2 < 0) { 29 | error(1, errno, "listen failed "); 30 | } 31 | 32 | // signal(SIGPIPE, SIG_IGN); 33 | 34 | return acceptor1; 35 | } -------------------------------------------------------------------------------- /channel.cpp: -------------------------------------------------------------------------------- 1 | #include "channel.h" 2 | #include "event_loop.h" 3 | 4 | struct channel * 5 | channel_new(int fd, int events, event_read_callback eventReadCallback, event_write_callback eventWriteCallback, 6 | void *data) { 7 | struct channel *chan = (struct channel *)malloc(sizeof(struct channel)); 8 | chan->fd = fd; 9 | chan->events = events; 10 | chan->eventReadCallback = eventReadCallback; 11 | chan->eventWriteCallback = eventWriteCallback; 12 | chan->data = data; 13 | return chan; 14 | } 15 | 16 | int channel_write_event_is_enabled(struct channel *channel) { 17 | return channel->events & EVENT_WRITE; 18 | } 19 | 20 | int channel_write_event_enable(struct channel *channel) { 21 | struct event_loop *eventLoop = (struct event_loop *) channel->data; 22 | channel->events = channel->events | EVENT_WRITE; 23 | event_loop_update_channel_event(eventLoop, channel->fd, channel); 24 | } 25 | 26 | int channel_write_event_disable(struct channel *channel) { 27 | struct event_loop *eventLoop = (struct event_loop *) channel->data; 28 | channel->events = channel->events & ~EVENT_WRITE; 29 | event_loop_update_channel_event(eventLoop, channel->fd, channel); 30 | } -------------------------------------------------------------------------------- /channel_map.cpp: -------------------------------------------------------------------------------- 1 | #include "channel_map.h" 2 | #include 3 | #include 4 | 5 | int map_make_space(struct channel_map *map, int slot, int msize) { 6 | if (map->nentries <= slot) { 7 | int nentries = map->nentries ? map->nentries : 32; 8 | void **tmp; 9 | 10 | while (nentries <= slot) 11 | nentries <<= 1; 12 | 13 | tmp = (void **) realloc(map->entries, nentries * msize); 14 | if (tmp == NULL) 15 | return (-1); 16 | 17 | memset(&tmp[map->nentries], 0, 18 | (nentries - map->nentries) * msize); 19 | 20 | map->nentries = nentries; 21 | map->entries = tmp; 22 | } 23 | 24 | return (0); 25 | } 26 | 27 | void map_init(struct channel_map *map) { 28 | map->nentries = 0; 29 | map->entries = NULL; 30 | } 31 | 32 | void map_clear(struct channel_map *map) { 33 | if (map->entries != NULL) { 34 | int i; 35 | for (i = 0; i < map->nentries; ++i) { 36 | if (map->entries[i] != NULL) 37 | free(map->entries[i]); 38 | } 39 | free(map->entries); 40 | map->entries = NULL; 41 | } 42 | map->nentries = 0; 43 | } 44 | 45 | 46 | -------------------------------------------------------------------------------- /tcp_server.h: -------------------------------------------------------------------------------- 1 | #ifndef TCP_SERVER_H 2 | #define TCP_SERVER_H 3 | 4 | #include "tcp_connection.h" 5 | #include "acceptor.h" 6 | #include "event_loop.h" 7 | #include "event_loop_thread_pool.h" 8 | 9 | struct TCPserver { 10 | int port; 11 | struct event_loop *eventLoop; 12 | struct acceptor *acceptor; 13 | connection_completed_call_back connectionCompletedCallBack; 14 | message_call_back messageCallBack; 15 | write_completed_call_back writeCompletedCallBack; 16 | connection_closed_call_back connectionClosedCallBack; 17 | int threadNum; 18 | struct event_loop_thread_pool *threadPool; 19 | void * data; //for callback use: http_server 20 | }; 21 | 22 | //准备监听套接字 23 | struct TCPserver * 24 | tcp_server_init(struct event_loop *eventLoop, struct acceptor *acceptor, 25 | connection_completed_call_back connectionCallBack, 26 | message_call_back messageCallBack, 27 | write_completed_call_back writeCompletedCallBack, 28 | connection_closed_call_back connectionClosedCallBack, 29 | int threadNum); 30 | 31 | //开启监听 32 | void tcp_server_start(struct TCPserver *tcpServer); 33 | 34 | //设置callback数据 35 | void tcp_server_set_data(struct TCPserver *tcpServer, void * data); 36 | 37 | void make_nonblocking(int fd); 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /common.h: -------------------------------------------------------------------------------- 1 | #ifndef __USER_COMMON_H_ 2 | #define __USER_COMMON_H_ 3 | 4 | #include /* basic system data types */ 5 | #include /* basic socket definitions */ 6 | #include /* timeval{} for select() */ 7 | #include /* timespec{} for pselect() */ 8 | #include /* sockaddr_in{} and other Internet defns */ 9 | #include /* inet(3) functions */ 10 | #include 11 | #include /* for nonblocking */ 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include /* for S_xxx file mode constants */ 18 | #include /* for iovec{} and readv/writev */ 19 | #include 20 | #include 21 | #include /* for Unix domain sockets */ 22 | 23 | #include /* for convenience */ 24 | #include 25 | #include /* for convenience */ 26 | #include /* for convenience */ 27 | #include 28 | #include 29 | 30 | 31 | #define EPOLL_ENABLE 1 /* epoll 从内核2.6开始完善支持，此处假设均支持 */ 32 | 33 | #ifdef EPOLL_ENABLE 34 | #include 35 | #endif 36 | 37 | #define SERV_PORT 12345 38 | #define MAXLINE 4096 39 | #define LISTENQ 1024 40 | 41 | 42 | #endif /* __USER_COMMON_H_ */ 43 | -------------------------------------------------------------------------------- /tcp_connection.h: -------------------------------------------------------------------------------- 1 | #ifndef TCP_CONNECTION 2 | #define TCP_CONNECTION 3 | 4 | #include "event_loop.h" 5 | #include "channel.h" 6 | #include "buffer.h" 7 | 8 | typedef int (*connection_completed_call_back)(struct tcp_connection *tcpConnection); 9 | 10 | typedef int (*message_call_back)(struct buffer *buffer, struct tcp_connection *tcpConnection); 11 | 12 | typedef int (*write_completed_call_back)(struct tcp_connection *tcpConnection); 13 | 14 | typedef int (*connection_closed_call_back)(struct tcp_connection *tcpConnection); 15 | 16 | struct tcp_connection { 17 | struct event_loop *eventLoop; 18 | struct channel *channel; 19 | char *name; 20 | struct buffer *input_buffer; //接收缓冲区 21 | struct buffer *output_buffer; //发送缓冲区 22 | 23 | connection_completed_call_back connectionCompletedCallBack; 24 | message_call_back messageCallBack; 25 | write_completed_call_back writeCompletedCallBack; 26 | connection_closed_call_back connectionClosedCallBack; 27 | 28 | void * data; //for callback use: http_server 29 | void * request; // for callback use 30 | void * response; // for callback use 31 | }; 32 | 33 | struct tcp_connection * 34 | tcp_connection_new(int fd, struct event_loop *eventLoop, connection_completed_call_back connectionCompletedCallBack, 35 | connection_closed_call_back connectionClosedCallBack, 36 | message_call_back messageCallBack, write_completed_call_back writeCompletedCallBack); 37 | 38 | //应用层调用入口 39 | int tcp_connection_send_data(struct tcp_connection *tcpConnection, void *data, int size); 40 | 41 | //应用层调用入口 42 | int tcp_connection_send_buffer(struct tcp_connection *tcpConnection, struct buffer * buffer); 43 | 44 | void tcp_connection_shutdown(struct tcp_connection * tcpConnection); 45 | //int tcp_connection_append_buffer(struct tcp_connection *tcpConnection); 46 | #endif -------------------------------------------------------------------------------- /event_loop_thread_pool.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "utils.h" 3 | #include "event_loop_thread_pool.h" 4 | 5 | struct event_loop_thread_pool *event_loop_thread_pool_new(struct event_loop *mainLoop, int threadNumber) { 6 | 7 | struct event_loop_thread_pool *threadPool = (struct event_loop_thread_pool *)malloc(sizeof(struct event_loop_thread_pool)); 8 | threadPool->mainLoop = mainLoop; 9 | threadPool->position = 0; 10 | threadPool->thread_number = threadNumber; 11 | threadPool->started = 0; 12 | threadPool->eventLoopThreads = NULL; 13 | return threadPool; 14 | } 15 | 16 | //一定是main thread发起 17 | void event_loop_thread_pool_start(struct event_loop_thread_pool *threadPool) { 18 | assert(!threadPool->started); 19 | assertInSameThread(threadPool->mainLoop); 20 | 21 | threadPool->started = 1; 22 | 23 | if (threadPool->thread_number <= 0) { 24 | return; 25 | } 26 | 27 | threadPool->eventLoopThreads = (struct event_loop_thread *)malloc(threadPool->thread_number * sizeof(struct event_loop_thread)); 28 | for (int i = 0; i < threadPool->thread_number; ++i) { 29 | event_loop_thread_init(&threadPool->eventLoopThreads[i], i); 30 | event_loop_thread_start(&threadPool->eventLoopThreads[i]); 31 | } 32 | } 33 | 34 | //一定是main thread中选择 35 | struct event_loop *event_loop_thread_pool_get_loop(struct event_loop_thread_pool *threadPool) { 36 | assert(threadPool->started); 37 | assertInSameThread(threadPool->mainLoop); 38 | 39 | //优先选择当前主线程 40 | struct event_loop *selected = threadPool->mainLoop; 41 | 42 | //从线程池中按照顺序挑选出一个线程 43 | if (threadPool->thread_number > 0) { 44 | selected = threadPool->eventLoopThreads[threadPool->position].eventLoop; 45 | if (++threadPool->position >= threadPool->thread_number) { 46 | threadPool->position = 0; 47 | } 48 | } 49 | 50 | return selected; 51 | } 52 | -------------------------------------------------------------------------------- /event_loop_thread.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "event_loop_thread.h" 3 | #include "event_loop.h" 4 | #include "log.h" 5 | 6 | void *event_loop_thread_run(void *arg) { 7 | struct event_loop_thread *eventLoopThread = (struct event_loop_thread *) arg; 8 | 9 | pthread_mutex_lock(&eventLoopThread->mutex); 10 | 11 | // 初始化化event loop，之后通知主线程 12 | eventLoopThread->eventLoop = event_loop_init_with_name(eventLoopThread->thread_name); 13 | yolanda_msgx("event loop thread init and signal, %s", eventLoopThread->thread_name); 14 | pthread_cond_signal(&eventLoopThread->cond); 15 | 16 | pthread_mutex_unlock(&eventLoopThread->mutex); 17 | 18 | //子线程event loop run 19 | event_loop_run(eventLoopThread->eventLoop); 20 | } 21 | 22 | //初始化已经分配内存的event_loop_thread 23 | int event_loop_thread_init(struct event_loop_thread *eventLoopThread, int i) { 24 | pthread_mutex_init(&eventLoopThread->mutex, NULL); 25 | pthread_cond_init(&eventLoopThread->cond, NULL); 26 | eventLoopThread->eventLoop = NULL; 27 | eventLoopThread->thread_count = 0; 28 | eventLoopThread->thread_tid = 0; 29 | 30 | char *buf = (char *)malloc(16); 31 | sprintf(buf, "Thread-%d\0", i + 1); 32 | eventLoopThread->thread_name = buf; 33 | 34 | return 0; 35 | } 36 | 37 | 38 | //由主线程调用，初始化一个子线程，并且让子线程开始运行event_loop 39 | struct event_loop *event_loop_thread_start(struct event_loop_thread *eventLoopThread) { 40 | pthread_create(&eventLoopThread->thread_tid, NULL, &event_loop_thread_run, eventLoopThread); 41 | 42 | assert(pthread_mutex_lock(&eventLoopThread->mutex) == 0); 43 | 44 | while (eventLoopThread->eventLoop == NULL) { 45 | assert(pthread_cond_wait(&eventLoopThread->cond, &eventLoopThread->mutex) == 0); 46 | } 47 | assert(pthread_mutex_unlock(&eventLoopThread->mutex) == 0); 48 | 49 | yolanda_msgx("event loop thread started, %s", eventLoopThread->thread_name); 50 | return eventLoopThread->eventLoop; 51 | } -------------------------------------------------------------------------------- /event_loop.h: -------------------------------------------------------------------------------- 1 | #ifndef EVENT_LOOP_H 2 | #define EVENT_LOOP_H 3 | 4 | #include 5 | #include "common.h" 6 | #include "channel.h" 7 | #include "channel_map.h" 8 | 9 | struct channel_element { 10 | int type; //1: add 2: delete 11 | struct channel *channel; 12 | struct channel_element *next; 13 | }; 14 | 15 | struct event_dispatcher; 16 | 17 | struct event_loop { 18 | int quit; 19 | const struct event_dispatcher *eventDispatcher; 20 | 21 | /** 对应的event_dispatcher的数据. */ 22 | void *event_dispatcher_data; 23 | struct channel_map *channelMap; 24 | 25 | int is_handle_pending; 26 | struct channel_element *pending_head; 27 | struct channel_element *pending_tail; 28 | 29 | pthread_t owner_thread_id; 30 | pthread_mutex_t mutex; 31 | pthread_cond_t cond; 32 | int socketPair[2]; 33 | char *thread_name; 34 | }; 35 | 36 | struct event_loop *event_loop_init(); 37 | 38 | struct event_loop *event_loop_init_with_name(char * thread_name); 39 | 40 | int event_loop_run(struct event_loop *eventLoop); 41 | 42 | void event_loop_wakeup(struct event_loop *eventLoop); 43 | 44 | int event_loop_add_channel_event(struct event_loop *eventLoop, int fd, struct channel *channel1); 45 | 46 | int event_loop_remove_channel_event(struct event_loop *eventLoop, int fd, struct channel *channel1); 47 | 48 | int event_loop_update_channel_event(struct event_loop *eventLoop, int fd, struct channel *channel1); 49 | 50 | int event_loop_handle_pending_add(struct event_loop *eventLoop, int fd, struct channel *channel); 51 | 52 | int event_loop_handle_pending_remove(struct event_loop *eventLoop, int fd, struct channel *channel); 53 | 54 | int event_loop_handle_pending_update(struct event_loop *eventLoop, int fd, struct channel *channel); 55 | 56 | // dispather派发完事件之后，调用该方法通知event_loop执行对应事件的相关callback方法 57 | // res: EVENT_READ | EVENT_READ等 58 | int channel_event_activate(struct event_loop *eventLoop, int fd, int res); 59 | 60 | #endif -------------------------------------------------------------------------------- /epoll_server_multithreads.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | #include "event_loop.h" 3 | #include "acceptor.h" 4 | #include "tcp_connection.h" 5 | #include "tcp_server.h" 6 | 7 | char rot13_char(char c) { 8 | if ((c >= 'a' && c <= 'm') || (c >= 'A' && c <= 'M')) 9 | return c + 13; 10 | else if ((c >= 'n' && c <= 'z') || (c >= 'N' && c <= 'Z')) 11 | return c - 13; 12 | else 13 | return c; 14 | } 15 | 16 | //连接建立之后的callback 17 | int onConnectionCompleted(struct tcp_connection *tcpConnection) { 18 | printf("connection completed\n"); 19 | return 0; 20 | } 21 | 22 | //数据读到buffer之后的callback 23 | int onMessage(struct buffer *input, struct tcp_connection *tcpConnection) { 24 | printf("get message from tcp connection %s\n", tcpConnection->name); 25 | printf("%s", input->data); 26 | 27 | struct buffer *output = buffer_new(); 28 | int size = buffer_readable_size(input); 29 | for (int i = 0; i < size; i++) { 30 | buffer_append_char(output, rot13_char(buffer_read_char(input))); 31 | } 32 | tcp_connection_send_buffer(tcpConnection, output); 33 | return 0; 34 | } 35 | 36 | //数据通过buffer写完之后的callback 37 | int onWriteCompleted(struct tcp_connection *tcpConnection) { 38 | printf("write completed\n"); 39 | return 0; 40 | } 41 | 42 | //连接关闭之后的callback 43 | int onConnectionClosed(struct tcp_connection *tcpConnection) { 44 | printf("connection closed\n"); 45 | return 0; 46 | } 47 | 48 | int main(int c, char **v) { 49 | //主线程event_loop 50 | struct event_loop *eventLoop = event_loop_init(); 51 | 52 | //初始化acceptor 53 | struct acceptor *acceptor = acceptor_init(SERV_PORT); 54 | 55 | //初始tcp_server，可以指定线程数目，这里线程是4，说明是一个acceptor线程，4个I/O线程，没一个I/O线程 56 | //tcp_server自己带一个event_loop 57 | struct TCPserver *tcpServer = tcp_server_init(eventLoop, acceptor, onConnectionCompleted, onMessage, 58 | onWriteCompleted, onConnectionClosed, 4); 59 | tcp_server_start(tcpServer); 60 | 61 | // main thread for acceptor 62 | event_loop_run(eventLoop); 63 | } -------------------------------------------------------------------------------- /log.cpp: -------------------------------------------------------------------------------- 1 | #include "log.h" 2 | #include 3 | #include 4 | #include /* for syslog() */ 5 | #include 6 | #include 7 | 8 | # define MAXLINE 4096 9 | 10 | 11 | /* error - print a diagnostic and optionally exit */ 12 | void error(int status, int err, char *fmt, ...) { 13 | va_list ap; 14 | 15 | va_start(ap, fmt); 16 | vfprintf(stderr, fmt, ap); 17 | va_end(ap); 18 | if (err) 19 | fprintf(stderr, ": %s (%d)\n", strerror(err), err); 20 | if (status) 21 | exit(status); 22 | } 23 | 24 | 25 | static void 26 | err_doit(int errnoflag, int level, const char *fmt, va_list ap) { 27 | int errno_save, n; 28 | char buf[MAXLINE + 1]; 29 | 30 | errno_save = errno; /* value caller might want printed */ 31 | 32 | vsnprintf(buf, MAXLINE, fmt, ap); /* safe */ 33 | 34 | n = strlen(buf); 35 | if (errnoflag) 36 | snprintf(buf + n, MAXLINE - n, ": %s", strerror(errno_save)); 37 | strcat(buf, "\n"); 38 | 39 | 40 | fflush(stdout); /* in case stdout and stderr are the same */ 41 | fputs(buf, stderr); 42 | fflush(stderr); 43 | 44 | return; 45 | } 46 | 47 | //void 48 | //err_quit(const char *fmt, ...) { 49 | // va_list ap; 50 | // 51 | // va_start(ap, fmt); 52 | // err_doit(0, LOG_ERR, fmt, ap); 53 | // va_end(ap); 54 | // exit(1); 55 | //} 56 | 57 | void yolanda_log(int severity, const char *msg) { 58 | const char *severity_str; 59 | switch (severity) { 60 | case LOG_DEBUG_TYPE: 61 | severity_str = "debug"; 62 | break; 63 | case LOG_MSG_TYPE: 64 | severity_str = "msg"; 65 | break; 66 | case LOG_WARN_TYPE: 67 | severity_str = "warn"; 68 | break; 69 | case LOG_ERR_TYPE: 70 | severity_str = "err"; 71 | break; 72 | default: 73 | severity_str = "???"; 74 | break; 75 | } 76 | (void) fprintf(stdout, "[%s] %s\n", severity_str, msg); 77 | 78 | } 79 | 80 | void yolanda_logx(int severity, const char *errstr, const char *fmt, va_list ap) 81 | { 82 | char buf[1024]; 83 | size_t len; 84 | 85 | if (fmt != NULL) 86 | vsnprintf(buf, sizeof(buf), fmt, ap); 87 | else 88 | buf[0] = '\0'; 89 | 90 | if (errstr) { 91 | len = strlen(buf); 92 | if (len < sizeof(buf) - 3) { 93 | snprintf(buf + len, sizeof(buf) - len, ": %s", errstr); 94 | } 95 | } 96 | 97 | yolanda_log(severity, buf); 98 | } 99 | 100 | void yolanda_msgx(const char *fmt, ...) 101 | { 102 | va_list ap; 103 | 104 | va_start(ap, fmt); 105 | yolanda_logx(LOG_MSG_TYPE, NULL, fmt, ap); 106 | va_end(ap); 107 | } 108 | 109 | void yolanda_debugx(const char *fmt, ...) 110 | { 111 | va_list ap; 112 | 113 | va_start(ap, fmt); 114 | yolanda_logx(LOG_DEBUG_TYPE, NULL, fmt, ap); 115 | va_end(ap); 116 | } -------------------------------------------------------------------------------- /buffer.cpp: -------------------------------------------------------------------------------- 1 | #include "buffer.h" 2 | #include "common.h" 3 | #include 4 | #include 5 | 6 | const char *CRLF = "\r\n"; 7 | 8 | struct buffer *buffer_new() { 9 | struct buffer *buffer1 = (struct buffer *)malloc(sizeof(struct buffer)); 10 | if (!buffer1) 11 | return NULL; 12 | 13 | buffer1->data = (char *)malloc(INIT_BUFFER_SIZE); 14 | buffer1->total_size = INIT_BUFFER_SIZE; 15 | buffer1->readIndex = 0; 16 | buffer1->writeIndex = 0; 17 | return buffer1; 18 | } 19 | 20 | void buffer_free(struct buffer *buffer1) { 21 | free(buffer1->data); 22 | free(buffer1); 23 | } 24 | 25 | int buffer_writeable_size(struct buffer *buffer) { 26 | return buffer->total_size - buffer->writeIndex; 27 | } 28 | 29 | int buffer_readable_size(struct buffer *buffer) { 30 | return buffer->writeIndex - buffer->readIndex; 31 | } 32 | 33 | int buffer_front_spare_size(struct buffer *buffer) { 34 | return buffer->readIndex; 35 | } 36 | 37 | void make_room(struct buffer *buffer, int size) { 38 | if (buffer_writeable_size(buffer) >= size) { 39 | return; 40 | } 41 | //如果front_spare和writeable的大小加起来可以容纳数据，则把可读数据往前面拷贝 42 | if (buffer_front_spare_size(buffer) + buffer_writeable_size(buffer) >= size) { 43 | int readable = buffer_readable_size(buffer); 44 | int i; 45 | for (i = 0; i < readable; i++) { 46 | memcpy(buffer->data + i, buffer->data + buffer->readIndex + i, 1); 47 | } 48 | buffer->readIndex = 0; 49 | buffer->writeIndex = readable; 50 | } else { 51 | //扩大缓冲区 52 | void *tmp = realloc(buffer->data, buffer->total_size + size); 53 | if (tmp == NULL) { 54 | return; 55 | } 56 | buffer->data = (char *)tmp; 57 | buffer->total_size += size; 58 | } 59 | } 60 | 61 | int buffer_append(struct buffer *buffer, void *data, int size) { 62 | if (data != NULL) { 63 | make_room(buffer, size); 64 | //拷贝数据到可写空间中 65 | memcpy(buffer->data + buffer->writeIndex, data, size); 66 | buffer->writeIndex += size; 67 | } 68 | } 69 | 70 | int buffer_append_char(struct buffer *buffer, char data) { 71 | make_room(buffer, 1); 72 | //拷贝数据到可写空间中 73 | buffer->data[buffer->writeIndex++] = data; 74 | } 75 | 76 | int buffer_append_string(struct buffer *buffer, char *data) { 77 | if (data != NULL) { 78 | int size = strlen(data); 79 | buffer_append(buffer, data, size); 80 | } 81 | } 82 | 83 | 84 | int buffer_socket_read(struct buffer *buffer, int fd) { 85 | char additional_buffer[INIT_BUFFER_SIZE]; 86 | struct iovec vec[2]; 87 | int max_writable = buffer_writeable_size(buffer); 88 | vec[0].iov_base = buffer->data + buffer->writeIndex; 89 | vec[0].iov_len = max_writable; 90 | vec[1].iov_base = additional_buffer; 91 | vec[1].iov_len = sizeof(additional_buffer); 92 | int result = readv(fd, vec, 2); 93 | if (result < 0) { 94 | return -1; 95 | } else if (result <= max_writable) { 96 | buffer->writeIndex += result; 97 | } else { 98 | buffer->writeIndex = buffer->total_size; 99 | buffer_append(buffer, additional_buffer, result - max_writable); 100 | } 101 | return result; 102 | } 103 | 104 | char buffer_read_char(struct buffer *buffer) { 105 | char c = buffer->data[buffer->readIndex]; 106 | buffer->readIndex++; 107 | return c; 108 | } 109 | 110 | char *buffer_find_CRLF(struct buffer *buffer) { 111 | char *crlf = (char *)memmem(buffer->data + buffer->readIndex, buffer_readable_size(buffer), CRLF, 2); 112 | return crlf; 113 | } -------------------------------------------------------------------------------- /tcp_connection.cpp: -------------------------------------------------------------------------------- 1 | #include "tcp_connection.h" 2 | #include "utils.h" 3 | #include "log.h" 4 | 5 | int handle_connection_closed(struct tcp_connection *tcpConnection) { 6 | struct event_loop *eventLoop = tcpConnection->eventLoop; 7 | struct channel *channel = tcpConnection->channel; 8 | event_loop_remove_channel_event(eventLoop, channel->fd, channel); 9 | if (tcpConnection->connectionClosedCallBack != NULL) { 10 | tcpConnection->connectionClosedCallBack(tcpConnection); 11 | } 12 | } 13 | 14 | int handle_read(void *data) { 15 | struct tcp_connection *tcpConnection = (struct tcp_connection *) data; 16 | struct buffer *input_buffer = tcpConnection->input_buffer; 17 | struct channel *channel = tcpConnection->channel; 18 | 19 | if (buffer_socket_read(input_buffer, channel->fd) > 0) { 20 | //应用程序真正读取Buffer里的数据 21 | if (tcpConnection->messageCallBack != NULL) { 22 | tcpConnection->messageCallBack(input_buffer, tcpConnection); 23 | } 24 | } else { 25 | handle_connection_closed(tcpConnection); 26 | } 27 | } 28 | 29 | //发送缓冲区可以往外写 30 | //把channel对应的output_buffer不断往外发送 31 | int handle_write(void *data) { 32 | struct tcp_connection *tcpConnection = (struct tcp_connection *) data; 33 | struct event_loop *eventLoop = tcpConnection->eventLoop; 34 | assertInSameThread(eventLoop); 35 | 36 | struct buffer *output_buffer = tcpConnection->output_buffer; 37 | struct channel *channel = tcpConnection->channel; 38 | 39 | ssize_t nwrited = write(channel->fd, output_buffer->data + output_buffer->readIndex, 40 | buffer_readable_size(output_buffer)); 41 | if (nwrited > 0) { 42 | //已读nwrited字节 43 | output_buffer->readIndex += nwrited; 44 | //如果数据完全发送出去，就不需要继续了 45 | if (buffer_readable_size(output_buffer) == 0) { 46 | channel_write_event_disable(channel); 47 | } 48 | //回调writeCompletedCallBack 49 | if (tcpConnection->writeCompletedCallBack != NULL) { 50 | tcpConnection->writeCompletedCallBack(tcpConnection); 51 | } 52 | } else { 53 | yolanda_msgx("handle_write for tcp connection %s", tcpConnection->name); 54 | } 55 | 56 | } 57 | 58 | struct tcp_connection * 59 | tcp_connection_new(int connected_fd, struct event_loop *eventLoop, 60 | connection_completed_call_back connectionCompletedCallBack, 61 | connection_closed_call_back connectionClosedCallBack, 62 | message_call_back messageCallBack, write_completed_call_back writeCompletedCallBack) { 63 | struct tcp_connection *tcpConnection = (struct tcp_connection *)malloc(sizeof(struct tcp_connection)); 64 | tcpConnection->writeCompletedCallBack = writeCompletedCallBack; 65 | tcpConnection->messageCallBack = messageCallBack; 66 | tcpConnection->connectionCompletedCallBack = connectionCompletedCallBack; 67 | tcpConnection->connectionClosedCallBack = connectionClosedCallBack; 68 | tcpConnection->eventLoop = eventLoop; 69 | tcpConnection->input_buffer = buffer_new(); 70 | tcpConnection->output_buffer = buffer_new(); 71 | 72 | 73 | char *buf = (char *)malloc(16); 74 | sprintf(buf, "connection-%d\0", connected_fd); 75 | tcpConnection->name = buf; 76 | 77 | // add event read for the new connection 78 | struct channel *channel1 = channel_new(connected_fd, EVENT_READ, handle_read, handle_write, tcpConnection); 79 | tcpConnection->channel = channel1; 80 | 81 | //connectionCompletedCallBack callback 82 | if (tcpConnection->connectionCompletedCallBack != NULL) { 83 | tcpConnection->connectionCompletedCallBack(tcpConnection); 84 | } 85 | 86 | event_loop_add_channel_event(tcpConnection->eventLoop, connected_fd, tcpConnection->channel); 87 | return tcpConnection; 88 | } 89 | 90 | //应用层调用入口 91 | int tcp_connection_send_data(struct tcp_connection *tcpConnection, void *data, int size) { 92 | size_t nwrited = 0; 93 | size_t nleft = size; 94 | int fault = 0; 95 | 96 | struct channel *channel = tcpConnection->channel; 97 | struct buffer *output_buffer = tcpConnection->output_buffer; 98 | 99 | //先往套接字尝试发送数据 100 | if (!channel_write_event_is_enabled(channel) && buffer_readable_size(output_buffer) == 0) { 101 | nwrited = write(channel->fd, data, size); 102 | if (nwrited >= 0) { 103 | nleft = nleft - nwrited; 104 | } else { 105 | nwrited = 0; 106 | if (errno != EWOULDBLOCK) { 107 | if (errno == EPIPE || errno == ECONNRESET) { 108 | fault = 1; 109 | } 110 | } 111 | } 112 | } 113 | 114 | if (!fault && nleft > 0) { 115 | //拷贝到Buffer中，Buffer的数据由框架接管 116 | buffer_append(output_buffer, data + nwrited, nleft); 117 | if (!channel_write_event_is_enabled(channel)) { 118 | channel_write_event_enable(channel); 119 | } 120 | } 121 | 122 | return nwrited; 123 | } 124 | 125 | int tcp_connection_send_buffer(struct tcp_connection *tcpConnection, struct buffer *buffer) { 126 | int size = buffer_readable_size(buffer); 127 | int result = tcp_connection_send_data(tcpConnection, buffer->data + buffer->readIndex, size); 128 | buffer->readIndex += size; 129 | return result; 130 | } 131 | 132 | void tcp_connection_shutdown(struct tcp_connection *tcpConnection) { 133 | if (shutdown(tcpConnection->channel->fd, SHUT_WR) < 0) { 134 | yolanda_msgx("tcp_connection_shutdown failed, socket == %d", tcpConnection->channel->fd); 135 | } 136 | } -------------------------------------------------------------------------------- /event_dispatcher.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "event_dispatcher.h" 3 | #include "event_loop.h" 4 | #include "log.h" 5 | 6 | #define MAXEVENTS 128 7 | 8 | typedef struct { 9 | int event_count; 10 | int nfds; 11 | int realloc_copy; 12 | int efd; 13 | struct epoll_event *events; 14 | } epoll_dispatcher_data; 15 | 16 | 17 | static void *epoll_init(struct event_loop *); 18 | 19 | static int epoll_add(struct event_loop *, struct channel *channel1); 20 | 21 | static int epoll_del(struct event_loop *, struct channel *channel1); 22 | 23 | static int epoll_update(struct event_loop *, struct channel *channel1); 24 | 25 | static int epoll_dispatch(struct event_loop *, struct timeval *); 26 | 27 | static void epoll_clear(struct event_loop *); 28 | 29 | const struct event_dispatcher epoll_dispatcher = { 30 | "epoll", 31 | epoll_init, 32 | epoll_add, 33 | epoll_del, 34 | epoll_update, 35 | epoll_dispatch, 36 | epoll_clear, 37 | }; 38 | 39 | void *epoll_init(struct event_loop *eventLoop) { 40 | epoll_dispatcher_data *epollDispatcherData = (epoll_dispatcher_data *)malloc(sizeof(epoll_dispatcher_data)); 41 | 42 | epollDispatcherData->event_count = 0; 43 | epollDispatcherData->nfds = 0; 44 | epollDispatcherData->realloc_copy = 0; 45 | epollDispatcherData->efd = 0; 46 | 47 | epollDispatcherData->efd = epoll_create1(0); 48 | if (epollDispatcherData->efd == -1) { 49 | error(1, errno, "epoll create failed"); 50 | } 51 | 52 | epollDispatcherData->events = (struct epoll_event *)calloc(MAXEVENTS, sizeof(struct epoll_event)); 53 | 54 | return epollDispatcherData; 55 | } 56 | 57 | 58 | int epoll_add(struct event_loop *eventLoop, struct channel *channel1) { 59 | epoll_dispatcher_data *pollDispatcherData = (epoll_dispatcher_data *) eventLoop->event_dispatcher_data; 60 | 61 | int fd = channel1->fd; 62 | int events = 0; 63 | if (channel1->events & EVENT_READ) { 64 | events = events | EPOLLIN; 65 | } 66 | if (channel1->events & EVENT_WRITE) { 67 | events = events | EPOLLOUT; 68 | } 69 | 70 | struct epoll_event event; 71 | event.data.fd = fd; 72 | event.events = events; 73 | // event.events = events | EPOLLET; 74 | if (epoll_ctl(pollDispatcherData->efd, EPOLL_CTL_ADD, fd, &event) == -1) { 75 | error(1, errno, "epoll_ctl add fd failed"); 76 | } 77 | 78 | return 0; 79 | } 80 | 81 | int epoll_del(struct event_loop *eventLoop, struct channel *channel1) { 82 | epoll_dispatcher_data *pollDispatcherData = (epoll_dispatcher_data *) eventLoop->event_dispatcher_data; 83 | 84 | int fd = channel1->fd; 85 | 86 | int events = 0; 87 | if (channel1->events & EVENT_READ) { 88 | events = events | EPOLLIN; 89 | } 90 | 91 | if (channel1->events & EVENT_WRITE) { 92 | events = events | EPOLLOUT; 93 | } 94 | 95 | struct epoll_event event; 96 | event.data.fd = fd; 97 | event.events = events; 98 | // event.events = events | EPOLLET; 99 | if (epoll_ctl(pollDispatcherData->efd, EPOLL_CTL_DEL, fd, &event) == -1) { 100 | error(1, errno, "epoll_ctl delete fd failed"); 101 | } 102 | 103 | return 0; 104 | } 105 | 106 | int epoll_update(struct event_loop *eventLoop, struct channel *channel1) { 107 | epoll_dispatcher_data *pollDispatcherData = (epoll_dispatcher_data *) eventLoop->event_dispatcher_data; 108 | 109 | int fd = channel1->fd; 110 | 111 | int events = 0; 112 | if (channel1->events & EVENT_READ) { 113 | events = events | EPOLLIN; 114 | } 115 | 116 | if (channel1->events & EVENT_WRITE) { 117 | events = events | EPOLLOUT; 118 | } 119 | 120 | struct epoll_event event; 121 | event.data.fd = fd; 122 | event.events = events; 123 | // event.events = events | EPOLLET; 124 | if (epoll_ctl(pollDispatcherData->efd, EPOLL_CTL_MOD, fd, &event) == -1) { 125 | error(1, errno, "epoll_ctl modify fd failed"); 126 | } 127 | 128 | return 0; 129 | } 130 | 131 | int epoll_dispatch(struct event_loop *eventLoop, struct timeval *timeval) { 132 | epoll_dispatcher_data *epollDispatcherData = (epoll_dispatcher_data *) eventLoop->event_dispatcher_data; 133 | int i, n; 134 | 135 | n = epoll_wait(epollDispatcherData->efd, epollDispatcherData->events, MAXEVENTS, -1); 136 | yolanda_msgx("epoll_wait wakeup, %s", eventLoop->thread_name); 137 | for (i = 0; i < n; i++) { 138 | if ((epollDispatcherData->events[i].events & EPOLLERR) || (epollDispatcherData->events[i].events & EPOLLHUP)) { 139 | fprintf(stderr, "epoll error\n"); 140 | close(epollDispatcherData->events[i].data.fd); 141 | continue; 142 | } 143 | 144 | if (epollDispatcherData->events[i].events & EPOLLIN) { 145 | yolanda_msgx("get message channel fd==%d for read, %s", epollDispatcherData->events[i].data.fd, eventLoop->thread_name); 146 | channel_event_activate(eventLoop, epollDispatcherData->events[i].data.fd, EVENT_READ); 147 | } 148 | 149 | if (epollDispatcherData->events[i].events & EPOLLOUT) { 150 | yolanda_msgx("get message channel fd==%d for write, %s", epollDispatcherData->events[i].data.fd,eventLoop->thread_name); 151 | channel_event_activate(eventLoop, epollDispatcherData->events[i].data.fd, EVENT_WRITE); 152 | } 153 | } 154 | 155 | return 0; 156 | } 157 | 158 | void epoll_clear(struct event_loop *eventLoop) { 159 | epoll_dispatcher_data *epollDispatcherData = (epoll_dispatcher_data *) eventLoop->event_dispatcher_data; 160 | 161 | free(epollDispatcherData->events); 162 | close(epollDispatcherData->efd); 163 | free(epollDispatcherData); 164 | eventLoop->event_dispatcher_data = NULL; 165 | 166 | return; 167 | } -------------------------------------------------------------------------------- /tcp_server.cpp: -------------------------------------------------------------------------------- 1 | #include "tcp_server.h" 2 | #include 3 | #include "common.h" 4 | #include "log.h" 5 | //#include "utils.h" 6 | // #include "event_loop_thread_pool.h" 7 | 8 | 9 | int tcp_server(int port) { 10 | int listenfd; 11 | listenfd = socket(AF_INET, SOCK_STREAM, 0); 12 | 13 | struct sockaddr_in server_addr; 14 | bzero(&server_addr, sizeof(server_addr)); 15 | server_addr.sin_family = AF_INET; 16 | server_addr.sin_addr.s_addr = htonl(INADDR_ANY); 17 | server_addr.sin_port = htons(port); 18 | 19 | int on = 1; 20 | setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)); 21 | 22 | int rt1 = bind(listenfd, (struct sockaddr *) &server_addr, sizeof(server_addr)); 23 | if (rt1 < 0) { 24 | error(1, errno, "bind failed "); 25 | } 26 | 27 | int rt2 = listen(listenfd, LISTENQ); 28 | if (rt2 < 0) { 29 | error(1, errno, "listen failed "); 30 | } 31 | 32 | signal(SIGPIPE, SIG_IGN); 33 | 34 | int connfd; 35 | struct sockaddr_in client_addr; 36 | socklen_t client_len = sizeof(client_addr); 37 | 38 | if ((connfd = accept(listenfd, (struct sockaddr *) &client_addr, &client_len)) < 0) { 39 | error(1, errno, "bind failed "); 40 | } 41 | 42 | return connfd; 43 | } 44 | 45 | 46 | int tcp_server_listen(int port) { 47 | int listenfd; 48 | listenfd = socket(AF_INET, SOCK_STREAM, 0); 49 | 50 | struct sockaddr_in server_addr; 51 | bzero(&server_addr, sizeof(server_addr)); 52 | server_addr.sin_family = AF_INET; 53 | server_addr.sin_addr.s_addr = htonl(INADDR_ANY); 54 | server_addr.sin_port = htons(port); 55 | 56 | int on = 1; 57 | setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)); 58 | 59 | int rt1 = bind(listenfd, (struct sockaddr *) &server_addr, sizeof(server_addr)); 60 | if (rt1 < 0) { 61 | error(1, errno, "bind failed "); 62 | } 63 | 64 | int rt2 = listen(listenfd, LISTENQ); 65 | if (rt2 < 0) { 66 | error(1, errno, "listen failed "); 67 | } 68 | 69 | signal(SIGPIPE, SIG_IGN); 70 | 71 | return listenfd; 72 | } 73 | 74 | 75 | int tcp_nonblocking_server_listen(int port) { 76 | int listenfd; 77 | listenfd = socket(AF_INET, SOCK_STREAM, 0); 78 | 79 | make_nonblocking(listenfd); 80 | 81 | struct sockaddr_in server_addr; 82 | bzero(&server_addr, sizeof(server_addr)); 83 | server_addr.sin_family = AF_INET; 84 | server_addr.sin_addr.s_addr = htonl(INADDR_ANY); 85 | server_addr.sin_port = htons(port); 86 | 87 | int on = 1; 88 | setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)); 89 | 90 | int rt1 = bind(listenfd, (struct sockaddr *) &server_addr, sizeof(server_addr)); 91 | if (rt1 < 0) { 92 | error(1, errno, "bind failed "); 93 | } 94 | 95 | int rt2 = listen(listenfd, LISTENQ); 96 | if (rt2 < 0) { 97 | error(1, errno, "listen failed "); 98 | } 99 | 100 | signal(SIGPIPE, SIG_IGN); 101 | 102 | return listenfd; 103 | } 104 | 105 | void make_nonblocking(int fd) { 106 | fcntl(fd, F_SETFL, O_NONBLOCK); 107 | } 108 | 109 | 110 | struct TCPserver * 111 | tcp_server_init(struct event_loop *eventLoop, struct acceptor *acceptor, 112 | connection_completed_call_back connectionCompletedCallBack, 113 | message_call_back messageCallBack, 114 | write_completed_call_back writeCompletedCallBack, 115 | connection_closed_call_back connectionClosedCallBack, 116 | int threadNum) { 117 | struct TCPserver *tcpServer = (struct TCPserver *)malloc(sizeof(struct TCPserver)); 118 | tcpServer->eventLoop = eventLoop; 119 | tcpServer->acceptor = acceptor; 120 | tcpServer->connectionCompletedCallBack = connectionCompletedCallBack; 121 | tcpServer->messageCallBack = messageCallBack; 122 | tcpServer->writeCompletedCallBack = writeCompletedCallBack; 123 | tcpServer->connectionClosedCallBack = connectionClosedCallBack; 124 | tcpServer->threadNum = threadNum; 125 | tcpServer->threadPool = event_loop_thread_pool_new(eventLoop, threadNum); 126 | tcpServer->data = NULL; 127 | 128 | return tcpServer; 129 | } 130 | 131 | 132 | int handle_connection_established(void *data) { 133 | struct TCPserver *tcpServer = (struct TCPserver *) data; 134 | struct acceptor *acceptor = tcpServer->acceptor; 135 | int listenfd = acceptor->listen_fd; 136 | 137 | struct sockaddr_in client_addr; 138 | socklen_t client_len = sizeof(client_addr); 139 | int connected_fd = accept(listenfd, (struct sockaddr *) &client_addr, &client_len); 140 | make_nonblocking(connected_fd); 141 | 142 | yolanda_msgx("new connection established, socket == %d", connected_fd); 143 | 144 | // choose event loop from the thread pool 145 | struct event_loop *eventLoop = event_loop_thread_pool_get_loop(tcpServer->threadPool); 146 | 147 | // create a new tcp connection 148 | struct tcp_connection *tcpConnection = tcp_connection_new(connected_fd, eventLoop, 149 | tcpServer->connectionCompletedCallBack, 150 | tcpServer->connectionClosedCallBack, 151 | tcpServer->messageCallBack, 152 | tcpServer->writeCompletedCallBack); 153 | //for callback use 154 | if (tcpServer->data != NULL) { 155 | tcpConnection->data = tcpServer->data; 156 | } 157 | return 0; 158 | } 159 | 160 | 161 | //开启监听 162 | void tcp_server_start(struct TCPserver *tcpServer) { 163 | struct acceptor *acceptor = tcpServer->acceptor; 164 | struct event_loop *eventLoop = tcpServer->eventLoop; 165 | 166 | //开启多个线程 167 | event_loop_thread_pool_start(tcpServer->threadPool); 168 | 169 | //acceptor主线程， 同时把tcpServer作为参数传给channel对象 170 | struct channel *channel = channel_new(acceptor->listen_fd, EVENT_READ, handle_connection_established, NULL, \ 171 | tcpServer); 172 | event_loop_add_channel_event(eventLoop, channel->fd, channel); 173 | return; 174 | } 175 | 176 | //设置callback数据 177 | void tcp_server_set_data(struct TCPserver *tcpServer, void *data) { 178 | if (data != NULL) { 179 | tcpServer->data = data; 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /event_loop.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "event_loop.h" 3 | #include "event_dispatcher.h" 4 | #include "common.h" 5 | #include "log.h" 6 | #include "utils.h" 7 | 8 | // in the i/o thread 9 | int event_loop_handle_pending_channel(struct event_loop *eventLoop) { 10 | //get the lock 11 | pthread_mutex_lock(&eventLoop->mutex); 12 | eventLoop->is_handle_pending = 1; 13 | 14 | struct channel_element *channelElement = eventLoop->pending_head; 15 | while (channelElement != NULL) { 16 | //save into event_map 17 | struct channel *channel = channelElement->channel; 18 | int fd = channel->fd; 19 | if (channelElement->type == 1) { 20 | event_loop_handle_pending_add(eventLoop, fd, channel); 21 | } else if (channelElement->type == 2) { 22 | event_loop_handle_pending_remove(eventLoop, fd, channel); 23 | } else if (channelElement->type == 3) { 24 | event_loop_handle_pending_update(eventLoop, fd, channel); 25 | } 26 | channelElement = channelElement->next; 27 | } 28 | 29 | eventLoop->pending_head = eventLoop->pending_tail = NULL; 30 | eventLoop->is_handle_pending = 0; 31 | 32 | //release the lock 33 | pthread_mutex_unlock(&eventLoop->mutex); 34 | 35 | return 0; 36 | } 37 | 38 | void event_loop_channel_buffer_nolock(struct event_loop *eventLoop, int fd, struct channel *channel1, int type) { 39 | //add channel into the pending list 40 | struct channel_element *channelElement = (struct channel_element *)malloc(sizeof(struct channel_element)); 41 | channelElement->channel = channel1; 42 | channelElement->type = type; 43 | channelElement->next = NULL; 44 | //第一个元素 45 | if (eventLoop->pending_head == NULL) { 46 | eventLoop->pending_head = eventLoop->pending_tail = channelElement; 47 | } else { 48 | eventLoop->pending_tail->next = channelElement; 49 | eventLoop->pending_tail = channelElement; 50 | } 51 | } 52 | 53 | int event_loop_do_channel_event(struct event_loop *eventLoop, int fd, struct channel *channel1, int type) { 54 | //get the lock 55 | pthread_mutex_lock(&eventLoop->mutex); 56 | assert(eventLoop->is_handle_pending == 0); 57 | event_loop_channel_buffer_nolock(eventLoop, fd, channel1, type); 58 | //release the lock 59 | pthread_mutex_unlock(&eventLoop->mutex); 60 | if (!isInSameThread(eventLoop)) { 61 | event_loop_wakeup(eventLoop); 62 | } else { 63 | event_loop_handle_pending_channel(eventLoop); 64 | } 65 | 66 | return 0; 67 | 68 | } 69 | 70 | int event_loop_add_channel_event(struct event_loop *eventLoop, int fd, struct channel *channel1) { 71 | return event_loop_do_channel_event(eventLoop, fd, channel1, 1); 72 | } 73 | 74 | int event_loop_remove_channel_event(struct event_loop *eventLoop, int fd, struct channel *channel1) { 75 | return event_loop_do_channel_event(eventLoop, fd, channel1, 2); 76 | } 77 | 78 | int event_loop_update_channel_event(struct event_loop *eventLoop, int fd, struct channel *channel1) { 79 | return event_loop_do_channel_event(eventLoop, fd, channel1, 3); 80 | } 81 | 82 | // in the i/o thread 83 | int event_loop_handle_pending_add(struct event_loop *eventLoop, int fd, struct channel *channel) { 84 | yolanda_msgx("add channel fd == %d, %s", fd, eventLoop->thread_name); 85 | struct channel_map *map = eventLoop->channelMap; 86 | 87 | if (fd < 0) 88 | return 0; 89 | 90 | if (fd >= map->nentries) { 91 | if (map_make_space(map, fd, sizeof(struct channel *)) == -1) 92 | return (-1); 93 | } 94 | 95 | //第一次创建，增加 96 | if ((map)->entries[fd] == NULL) { 97 | map->entries[fd] = channel; 98 | //add channel 99 | struct event_dispatcher *eventDispatcher = (struct event_dispatcher *)eventLoop->eventDispatcher; 100 | eventDispatcher->add(eventLoop, channel); 101 | return 1; 102 | } 103 | 104 | return 0; 105 | } 106 | 107 | // in the i/o thread 108 | int event_loop_handle_pending_remove(struct event_loop *eventLoop, int fd, struct channel *channel1) { 109 | struct channel_map *map = eventLoop->channelMap; 110 | assert(fd == channel1->fd); 111 | 112 | if (fd < 0) 113 | return 0; 114 | 115 | if (fd >= map->nentries) 116 | return (-1); 117 | 118 | struct channel *channel2 = (struct channel *)map->entries[fd]; 119 | 120 | //update dispatcher(multi-thread)here 121 | struct event_dispatcher *eventDispatcher = (struct event_dispatcher *)eventLoop->eventDispatcher; 122 | 123 | int retval = 0; 124 | if (eventDispatcher->del(eventLoop, channel2) == -1) { 125 | retval = -1; 126 | } else { 127 | retval = 1; 128 | } 129 | 130 | map->entries[fd] = NULL; 131 | return retval; 132 | } 133 | 134 | // in the i/o thread 135 | int event_loop_handle_pending_update(struct event_loop *eventLoop, int fd, struct channel *channel) { 136 | yolanda_msgx("update channel fd == %d, %s", fd, eventLoop->thread_name); 137 | struct channel_map *map = eventLoop->channelMap; 138 | 139 | if (fd < 0) 140 | return 0; 141 | 142 | if ((map)->entries[fd] == NULL) { 143 | return (-1); 144 | } 145 | 146 | //update channel 147 | struct event_dispatcher *eventDispatcher = (struct event_dispatcher *)eventLoop->eventDispatcher; 148 | eventDispatcher->update(eventLoop, channel); 149 | } 150 | 151 | int channel_event_activate(struct event_loop *eventLoop, int fd, int revents) { 152 | struct channel_map *map = eventLoop->channelMap; 153 | yolanda_msgx("activate channel fd == %d, revents=%d, %s", fd, revents, eventLoop->thread_name); 154 | 155 | if (fd < 0) 156 | return 0; 157 | 158 | if (fd >= map->nentries)return (-1); 159 | 160 | struct channel *channel = (struct channel *)map->entries[fd]; 161 | assert(fd == channel->fd); 162 | 163 | if (revents & (EVENT_READ)) { 164 | if (channel->eventReadCallback) channel->eventReadCallback(channel->data); 165 | } 166 | if (revents & (EVENT_WRITE)) { 167 | if (channel->eventWriteCallback) channel->eventWriteCallback(channel->data); 168 | } 169 | 170 | return 0; 171 | } 172 | 173 | void event_loop_wakeup(struct event_loop *eventLoop) { 174 | char one = 'a'; 175 | ssize_t n = write(eventLoop->socketPair[0], &one, sizeof one); 176 | if (n != sizeof one) { 177 | LOG_ERR("wakeup event loop thread failed"); 178 | } 179 | } 180 | 181 | int handleWakeup(void *data) { 182 | struct event_loop *eventLoop = (struct event_loop *) data; 183 | char one; 184 | ssize_t n = read(eventLoop->socketPair[1], &one, sizeof one); 185 | if (n != sizeof one) { 186 | LOG_ERR("handleWakeup failed"); 187 | } 188 | yolanda_msgx("wakeup, %s", eventLoop->thread_name); 189 | } 190 | 191 | struct event_loop *event_loop_init() { 192 | return event_loop_init_with_name(NULL); 193 | } 194 | 195 | struct event_loop *event_loop_init_with_name(char *thread_name) { 196 | struct event_loop *eventLoop = (struct event_loop *)malloc(sizeof(struct event_loop)); 197 | pthread_mutex_init(&eventLoop->mutex, NULL); 198 | pthread_cond_init(&eventLoop->cond, NULL); 199 | 200 | if (thread_name != NULL) { 201 | eventLoop->thread_name = thread_name; 202 | } else { 203 | eventLoop->thread_name = "main thread"; 204 | } 205 | 206 | eventLoop->quit = 0; 207 | eventLoop->channelMap = (struct channel_map *)malloc(sizeof(struct channel_map)); 208 | map_init(eventLoop->channelMap); 209 | 210 | #ifdef EPOLL_ENABLE 211 | yolanda_msgx("set epoll as dispatcher, %s", eventLoop->thread_name); 212 | eventLoop->eventDispatcher = &epoll_dispatcher; 213 | #else 214 | yolanda_msgx("set poll as dispatcher, %s", eventLoop->thread_name); 215 | eventLoop->eventDispatcher = &poll_dispatcher; 216 | #endif 217 | eventLoop->event_dispatcher_data = eventLoop->eventDispatcher->init(eventLoop); 218 | 219 | //add the socketfd to event 220 | eventLoop->owner_thread_id = pthread_self(); 221 | if (socketpair(AF_UNIX, SOCK_STREAM, 0, eventLoop->socketPair) < 0) { 222 | LOG_ERR("socketpair set fialed"); 223 | } 224 | eventLoop->is_handle_pending = 0; 225 | eventLoop->pending_head = NULL; 226 | eventLoop->pending_tail = NULL; 227 | 228 | struct channel *channel = channel_new(eventLoop->socketPair[1], EVENT_READ, handleWakeup, NULL, 229 | eventLoop); 230 | event_loop_add_channel_event(eventLoop, eventLoop->socketPair[0], channel); 231 | 232 | return eventLoop; 233 | } 234 | 235 | /** 236 | * 237 | * 1.参数验证 238 | * 2.调用dispatcher来进行事件分发,分发完回调事件处理函数 239 | */ 240 | int event_loop_run(struct event_loop *eventLoop) { 241 | assert(eventLoop != NULL); 242 | 243 | struct event_dispatcher *dispatcher = (struct event_dispatcher *)eventLoop->eventDispatcher; 244 | 245 | if (eventLoop->owner_thread_id != pthread_self()) { 246 | exit(1); 247 | } 248 | 249 | yolanda_msgx("event loop run, %s", eventLoop->thread_name); 250 | struct timeval timeval; 251 | timeval.tv_sec = 1; 252 | 253 | while (!eventLoop->quit) { 254 | //block here to wait I/O event, and get active channels 255 | dispatcher->dispatch(eventLoop, &timeval); 256 | 257 | //handle the pending channel 258 | event_loop_handle_pending_channel(eventLoop); 259 | } 260 | 261 | yolanda_msgx("event loop end, %s", eventLoop->thread_name); 262 | return 0; 263 | } 264 | 265 | 266 | --------------------------------------------------------------------------------